From 62ae2d4540acd9c64a9d89daa62190e3faf73cdc Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 21 Oct 2024 10:26:54 +0300 Subject: [PATCH 1/5] Address review --- .github/workflows/build.yml | 2 +- .github/workflows/reusable-macos.yml | 2 +- .github/workflows/reusable-ubuntu.yml | 2 +- .github/workflows/reusable-windows.yml | 2 +- Makefile.pre.in | 2 +- PCbuild/rt.bat | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec7904c2e2cc73..fd168bd1d92877 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -430,7 +430,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test + run: xvfb-run make test TESTOPTS="-ugui" build_tsan: name: 'Thread sanitizer' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index b3a160fbbf8053..29fe2259bbbc71 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -69,4 +69,4 @@ jobs: --fail-on-improvement --path-prefix="./" - name: Tests - run: make test + run: make test TESTOPTS="-ugui" diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 0cf40ba8a9b03b..526f4765094af5 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -105,4 +105,4 @@ jobs: run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + run: xvfb-run make test TESTOPTS="-ugui" diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index dcfc62d7f5d145..c2de64af7b4462 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -43,5 +43,5 @@ jobs: run: >- .\PCbuild\rt.bat -p ${{ inputs.arch }} - -d -q --fast-ci + -d -q --fast-ci -ugui ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} diff --git a/Makefile.pre.in b/Makefile.pre.in index fb6f22d57397db..20621257575e4d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2097,7 +2097,7 @@ cleantest: all # Similar to buildbottest, but use --fast-ci option, instead of --slow-ci. .PHONY: test test: all - $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) + $(TESTRUNNER) --fast-ci -u-gui --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index c436215780fda0..06474d27c4b16f 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -34,7 +34,7 @@ set pyname=python set suffix= set qmode= set dashO= -set regrtestargs=--fast-ci +set regrtestargs=--fast-ci -u-gui set exe= :CheckOpts From 3bf42ab2cffec0cdb76f3d5cba20ebcbd287eff4 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 21 Oct 2024 14:13:33 +0300 Subject: [PATCH 2/5] Add `make ci` target and a NEWS entry --- .github/workflows/build.yml | 2 +- .github/workflows/reusable-macos.yml | 2 +- .github/workflows/reusable-ubuntu.yml | 2 +- Makefile.pre.in | 6 ++++++ .../Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst | 2 ++ 5 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fd168bd1d92877..870348ae0b9d94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -430,7 +430,7 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test TESTOPTS="-ugui" + run: xvfb-run make ci build_tsan: name: 'Thread sanitizer' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 29fe2259bbbc71..27a9d07b5953ae 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -69,4 +69,4 @@ jobs: --fail-on-improvement --path-prefix="./" - name: Tests - run: make test TESTOPTS="-ugui" + run: make ci diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 526f4765094af5..3eec7c51ee9872 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -105,4 +105,4 @@ jobs: run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test TESTOPTS="-ugui" + run: xvfb-run make ci diff --git a/Makefile.pre.in b/Makefile.pre.in index 20621257575e4d..c8e893f95daaca 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2099,6 +2099,12 @@ cleantest: all test: all $(TESTRUNNER) --fast-ci -u-gui --timeout=$(TESTTIMEOUT) $(TESTOPTS) +# Run a basic set of regression tests inside the CI. +# This excludes some tests that are particularly resource-intensive. +# Similar to test, but also runs GUI tests. +ci: all + $(MAKE) test TESTOPTS="-ugui" + # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. .PHONY: testuniversal diff --git a/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst b/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst new file mode 100644 index 00000000000000..061a1f9f27599c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-10-21-14-10-56.gh-issue-125730.kcWbvI.rst @@ -0,0 +1,2 @@ +Change ``make test`` to not run GUI tests by default. Use ``make ci`` to run +tests with GUI tests instead. From 37e95270251df8cd891d5b700b1e460153f3aab7 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 30 Oct 2024 12:27:27 +0300 Subject: [PATCH 3/5] Address review --- Doc/using/configure.rst | 8 +++++++- Makefile.pre.in | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 0e7b1be5b4bc2e..6004e13ad3a38d 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1142,13 +1142,19 @@ make test ^^^^^^^^^ Build the ``all`` target and run the Python test suite with the -``--fast-ci`` option. Variables: +``--fast-ci`` option without GUI tests. Variables: * ``TESTOPTS``: additional regrtest command-line options. * ``TESTPYTHONOPTS``: additional Python command-line options. * ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). +make ci +^^^^^^^ + +This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. + + make buildbottest ^^^^^^^^^^^^^^^^^ diff --git a/Makefile.pre.in b/Makefile.pre.in index 08b4aa460fa79c..aa7fa4e29d84c2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2110,7 +2110,7 @@ test: all # This excludes some tests that are particularly resource-intensive. # Similar to test, but also runs GUI tests. ci: all - $(MAKE) test TESTOPTS="-ugui" + $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. From d5ed9a117bf7a2cb584b4096cf70410b62c813c9 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 30 Oct 2024 12:48:06 +0300 Subject: [PATCH 4/5] Address review --- Doc/using/configure.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6004e13ad3a38d..83994af795e3fc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1154,6 +1154,8 @@ make ci This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. +.. versionadded:: 3.14 + make buildbottest ^^^^^^^^^^^^^^^^^ From 93d298201a33f6eeb60c3e4303d732a8402751cb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 30 Oct 2024 15:36:07 +0300 Subject: [PATCH 5/5] Revert windows changes --- .github/workflows/reusable-windows.yml | 2 +- PCbuild/rt.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index c2de64af7b4462..dcfc62d7f5d145 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -43,5 +43,5 @@ jobs: run: >- .\PCbuild\rt.bat -p ${{ inputs.arch }} - -d -q --fast-ci -ugui + -d -q --fast-ci ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 06474d27c4b16f..c436215780fda0 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -34,7 +34,7 @@ set pyname=python set suffix= set qmode= set dashO= -set regrtestargs=--fast-ci -u-gui +set regrtestargs=--fast-ci set exe= :CheckOpts