From 67542cc4f60b6b8d1c7dac49d7560f3ca6c82d9f Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Fri, 27 Dec 2024 22:28:09 -0800 Subject: [PATCH 1/9] TYP: use mypy_primer to surface type checking regressions Resolves #28054 --- .github/workflows/mypy_primer.yml | 97 ++++++++++++++++++++++ .github/workflows/mypy_primer_comment.yml | 98 +++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 .github/workflows/mypy_primer.yml create mode 100644 .github/workflows/mypy_primer_comment.yml diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml new file mode 100644 index 000000000000..8ee48d869773 --- /dev/null +++ b/.github/workflows/mypy_primer.yml @@ -0,0 +1,97 @@ +name: Run mypy_primer + +on: + # Only run on PR, since we diff against main + pull_request: + paths: + - "**/*.pyi" + - ".github/workflows/mypy_primer.yml" + - ".github/workflows/mypy_primer_comment.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + mypy_primer: + name: Run + runs-on: ubuntu-latest + strategy: + matrix: + shard-index: [0, 1, 2, 3] + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + path: numpy_to_test + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: pip install git+https://github.com/hauntsaninja/mypy_primer.git + - name: Run mypy_primer + shell: bash + run: | + cd numpy_to_test + MYPY_VERSION=$(grep mypy== requirements/test_requirements.txt | sed -n 's/mypy==\([^;]*\).*/\1/p') + + echo "new commit" + git checkout $GITHUB_SHA + git rev-list --format=%s --max-count=1 HEAD + + MERGE_BASE=$(git merge-base $GITHUB_SHA origin/$GITHUB_BASE_REF) + git worktree add ../numpy_base $MERGE_BASE + cd ../numpy_base + + echo "base commit" + git rev-list --format=%s --max-count=1 HEAD + + echo '' + cd .. + # fail action if exit code isn't zero or one + ( + mypy_primer \ + --new v${MYPY_VERSION} --old v${MYPY_VERSION} \ + --known-dependency-selector numpy \ + --old-prepend-path numpy_base --new-prepend-path numpy_to_test \ + --num-shards 4 --shard-index ${{ matrix.shard-index }} \ + --debug \ + --output concise \ + | tee diff_${{ matrix.shard-index }}.txt + ) || [ $? -eq 1 ] + - if: ${{ matrix.shard-index == 0 }} + name: Save PR number + run: | + echo ${{ github.event.pull_request.number }} | tee pr_number.txt + - name: Upload mypy_primer diff + PR number + uses: actions/upload-artifact@v4 + if: ${{ matrix.shard-index == 0 }} + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: | + diff_${{ matrix.shard-index }}.txt + pr_number.txt + - name: Upload mypy_primer diff + uses: actions/upload-artifact@v4 + if: ${{ matrix.shard-index != 0 }} + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: diff_${{ matrix.shard-index }}.txt + + join_artifacts: + name: Join artifacts + runs-on: ubuntu-latest + needs: [mypy_primer] + permissions: + contents: read + steps: + - name: Merge artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: mypy_primer_diffs + pattern: mypy_primer_diffs-* + delete-merged: true diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml new file mode 100644 index 000000000000..56909135c84e --- /dev/null +++ b/.github/workflows/mypy_primer_comment.yml @@ -0,0 +1,98 @@ +name: Comment with mypy_primer diff + +on: + workflow_run: + workflows: + - Run mypy_primer + types: + - completed + +permissions: + contents: read + pull-requests: write + +jobs: + comment: + name: Comment PR from mypy_primer + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Download diffs + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const [matchArtifact] = artifacts.data.artifacts.filter((artifact) => + artifact.name == "mypy_primer_diffs"); + + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }); + fs.writeFileSync("diff.zip", Buffer.from(download.data)); + + - run: unzip diff.zip + - run: | + cat diff_*.txt | tee fulldiff.txt + + - name: Post comment + id: post-comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const MAX_CHARACTERS = 50000 + const MAX_CHARACTERS_PER_PROJECT = MAX_CHARACTERS / 3 + + const fs = require('fs') + let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' }) + + function truncateIfNeeded(original, maxLength) { + if (original.length <= maxLength) { + return original + } + let truncated = original.substring(0, maxLength) + // further, remove last line that might be truncated + truncated = truncated.substring(0, truncated.lastIndexOf('\n')) + let lines_truncated = original.split('\n').length - truncated.split('\n').length + return `${truncated}\n\n... (truncated ${lines_truncated} lines) ...` + } + + const projects = data.split('\n\n') + // don't let one project dominate + data = projects.map(project => truncateIfNeeded(project, MAX_CHARACTERS_PER_PROJECT)).join('\n\n') + // posting comment fails if too long, so truncate + data = truncateIfNeeded(data, MAX_CHARACTERS) + + console.log("Diff from mypy_primer:") + console.log(data) + + let body + if (data.trim()) { + body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on type check results on a corpus of open source code:\n```diff\n' + data + '```' + } else { + body = "According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change doesn't affect type check results on a corpus of open source code. ✅" + } + const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + return prNumber + + - name: Hide old comments + # v0.4.0 + uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + leave_visible: 1 + issue_number: ${{ steps.post-comment.outputs.result }} From 3aa89c6832de067078467e549157af353a479287 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Fri, 27 Dec 2024 22:42:56 -0800 Subject: [PATCH 2/9] introduce temporary regression --- numpy/__init__.pyi | 2 -- 1 file changed, 2 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 02d3d5d1cc7a..0d31a70b2e8f 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -597,9 +597,7 @@ from numpy.lib._type_check_impl import ( mintypecode, real, imag, - iscomplex, isreal, - iscomplexobj, isrealobj, nan_to_num, real_if_close, From 2b86dfa7fc797d1025ad892ca212678a6f2fed13 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Fri, 27 Dec 2024 22:56:09 -0800 Subject: [PATCH 3/9] Revert "introduce temporary regression" This reverts commit 3aa89c6832de067078467e549157af353a479287. --- numpy/__init__.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 0d31a70b2e8f..02d3d5d1cc7a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -597,7 +597,9 @@ from numpy.lib._type_check_impl import ( mintypecode, real, imag, + iscomplex, isreal, + iscomplexobj, isrealobj, nan_to_num, real_if_close, From d48bac47176209630160029d02af5278db4be6b3 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Fri, 27 Dec 2024 22:58:06 -0800 Subject: [PATCH 4/9] add todo --- .github/workflows/mypy_primer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 8ee48d869773..b8f8d79036fb 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -53,6 +53,8 @@ jobs: echo '' cd .. # fail action if exit code isn't zero or one + # TODO: note that we don't build numpy, so if a project attempts to use the + # numpy mypy plugin, we may see some issues involving version skew. ( mypy_primer \ --new v${MYPY_VERSION} --old v${MYPY_VERSION} \ From 8299755ff26b713dc470aaee171cf7f61451c31e Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 28 Dec 2024 14:24:27 -0800 Subject: [PATCH 5/9] reduce shards --- .github/workflows/mypy_primer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index b8f8d79036fb..295cdec95a8e 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - shard-index: [0, 1, 2, 3] + shard-index: [0] fail-fast: false steps: - uses: actions/checkout@v4 From d75adc89fd0b6c1870c810a711de214f529bdd36 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 28 Dec 2024 14:29:46 -0800 Subject: [PATCH 6/9] only comment if there's a diff --- .github/workflows/mypy_primer_comment.yml | 31 ++++++++++------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml index 56909135c84e..89fda4279a84 100644 --- a/.github/workflows/mypy_primer_comment.yml +++ b/.github/workflows/mypy_primer_comment.yml @@ -17,6 +17,13 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: + - name: Hide old comments + # v0.4.0 + uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + issue_number: ${{ steps.post-comment.outputs.result }} + - name: Download diffs uses: actions/github-script@v7 with: @@ -77,22 +84,12 @@ jobs: let body if (data.trim()) { body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on type check results on a corpus of open source code:\n```diff\n' + data + '```' - } else { - body = "According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change doesn't affect type check results on a corpus of open source code. ✅" + const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) } - const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) - await github.rest.issues.createComment({ - issue_number: prNumber, - owner: context.repo.owner, - repo: context.repo.repo, - body - }) return prNumber - - - name: Hide old comments - # v0.4.0 - uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - leave_visible: 1 - issue_number: ${{ steps.post-comment.outputs.result }} From b8920c662b9f5cb31035f333c9a65693ac006fa2 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 28 Dec 2024 15:49:59 -0800 Subject: [PATCH 7/9] fix sharding and add comment --- .github/workflows/mypy_primer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 295cdec95a8e..5a687ab191ba 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - shard-index: [0] + shard-index: [0] # e.g. change this to [0, 1, 2] and --num-shards below to 3 fail-fast: false steps: - uses: actions/checkout@v4 @@ -60,7 +60,7 @@ jobs: --new v${MYPY_VERSION} --old v${MYPY_VERSION} \ --known-dependency-selector numpy \ --old-prepend-path numpy_base --new-prepend-path numpy_to_test \ - --num-shards 4 --shard-index ${{ matrix.shard-index }} \ + --num-shards 1 --shard-index ${{ matrix.shard-index }} \ --debug \ --output concise \ | tee diff_${{ matrix.shard-index }}.txt From f40d6cbbf8f3cba45d84a586c5311514ca92091a Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 28 Dec 2024 19:36:49 -0800 Subject: [PATCH 8/9] add hashes --- .github/workflows/mypy_primer.yml | 10 +++++----- .github/workflows/mypy_primer_comment.yml | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 5a687ab191ba..08e4b117d3c8 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -24,11 +24,11 @@ jobs: shard-index: [0] # e.g. change this to [0, 1, 2] and --num-shards below to 3 fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: numpy_to_test fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: "3.12" - name: Install dependencies @@ -70,7 +70,7 @@ jobs: run: | echo ${{ github.event.pull_request.number }} | tee pr_number.txt - name: Upload mypy_primer diff + PR number - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 if: ${{ matrix.shard-index == 0 }} with: name: mypy_primer_diffs-${{ matrix.shard-index }} @@ -78,7 +78,7 @@ jobs: diff_${{ matrix.shard-index }}.txt pr_number.txt - name: Upload mypy_primer diff - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 if: ${{ matrix.shard-index != 0 }} with: name: mypy_primer_diffs-${{ matrix.shard-index }} @@ -92,7 +92,7 @@ jobs: contents: read steps: - name: Merge artifacts - uses: actions/upload-artifact/merge@v4 + uses: actions/upload-artifact/merge@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: mypy_primer_diffs pattern: mypy_primer_diffs-* diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml index 89fda4279a84..711fd8867206 100644 --- a/.github/workflows/mypy_primer_comment.yml +++ b/.github/workflows/mypy_primer_comment.yml @@ -18,14 +18,13 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Hide old comments - # v0.4.0 - uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf + uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf # v0.4.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} issue_number: ${{ steps.post-comment.outputs.result }} - name: Download diffs - uses: actions/github-script@v7 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const fs = require('fs'); @@ -51,7 +50,7 @@ jobs: - name: Post comment id: post-comment - uses: actions/github-script@v7 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | From d127644f419079224af0ae57b0173d25237ba7f7 Mon Sep 17 00:00:00 2001 From: Shantanu Jain Date: Sat, 28 Dec 2024 19:36:53 -0800 Subject: [PATCH 9/9] split string --- .github/workflows/mypy_primer_comment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml index 711fd8867206..e484589ad446 100644 --- a/.github/workflows/mypy_primer_comment.yml +++ b/.github/workflows/mypy_primer_comment.yml @@ -82,7 +82,9 @@ jobs: let body if (data.trim()) { - body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on type check results on a corpus of open source code:\n```diff\n' + data + '```' + body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), ' + body += 'showing the effect of this PR on type check results on a corpus of open source code:\n```diff\n' + body += data + '```' const prNumber = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) await github.rest.issues.createComment({ issue_number: prNumber,