From 09d085a923f4e745de559c6bf26522122f3fbae9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 30 Apr 2024 20:57:59 -0700 Subject: [PATCH 1/4] experimenting to remove exclude prop --- .github/workflows/jit.yml | 8 ++------ .github/workflows/skip-tests.txt | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/skip-tests.txt diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b969e6d13e61a3..c5fb142948c193 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -75,14 +75,10 @@ jobs: architecture: aarch64 runner: ubuntu-latest compiler: gcc - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 runner: ubuntu-latest compiler: clang - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection env: CC: ${{ matrix.compiler }} steps: @@ -115,7 +111,7 @@ jobs: SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python.exe -m test --ignorefile=${{github.workspace}}/.github/workflows/skip-tests.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux @@ -146,4 +142,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --exclude --ignorefile=${{github.workspace}}/.github/workflows/skip-tests.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/skip-tests.txt b/.github/workflows/skip-tests.txt new file mode 100644 index 00000000000000..260531f22dcc06 --- /dev/null +++ b/.github/workflows/skip-tests.txt @@ -0,0 +1,18 @@ +test_pathlib +test_posixpath +test_unix_events +test_init +test_process_pool +test_shutdown +test_multiprocessing_fork +test_cmd_line +test_faulthandler +test_os +test_perf_profiler +test_posix +test_signal +test_socket +test_subprocess +test_threading +test_venv +test_external_inspection From 856c1c31842aef9c38f8fefb92dbe5a76916cdfb Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 2 May 2024 21:21:20 -0700 Subject: [PATCH 2/4] Add tests to exclude file --- .github/workflows/jit.yml | 7 +- .github/workflows/skip-tests.txt | 18 ------ Tools/jit/ignore-tests-emulated-linux.txt | 79 +++++++++++++++++++++++ 3 files changed, 83 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/skip-tests.txt create mode 100644 Tools/jit/ignore-tests-emulated-linux.txt diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c5fb142948c193..26284a0a0c9b3a 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -111,7 +111,7 @@ jobs: SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --ignorefile=${{github.workspace}}/.github/workflows/skip-tests.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux @@ -121,11 +121,12 @@ jobs: export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -142,4 +143,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude --ignorefile=${{github.workspace}}/.github/workflows/skip-tests.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --ignorefile=${{ github.workspace }}/Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/skip-tests.txt b/.github/workflows/skip-tests.txt deleted file mode 100644 index 260531f22dcc06..00000000000000 --- a/.github/workflows/skip-tests.txt +++ /dev/null @@ -1,18 +0,0 @@ -test_pathlib -test_posixpath -test_unix_events -test_init -test_process_pool -test_shutdown -test_multiprocessing_fork -test_cmd_line -test_faulthandler -test_os -test_perf_profiler -test_posix -test_signal -test_socket -test_subprocess -test_threading -test_venv -test_external_inspection diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt new file mode 100644 index 00000000000000..84e8c0ee8afedb --- /dev/null +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -0,0 +1,79 @@ +test_multiprocessing_fork +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess +test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling +test.test_cmd_line.CmdLineTest.test_no_std_streams +test.test_cmd_line.CmdLineTest.test_no_stdin +test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers +test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace +test.test_external_inspection.TestGetStackTrace.test_self_trace +test.test_faulthandler.FaultHandlerTests.test_enable_fd +test.test_faulthandler.FaultHandlerTests.test_enable_file +test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists +test.test_os.TimerfdTests.test_timerfd_initval +test.test_os.TimerfdTests.test_timerfd_interval +test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME +test.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.PosixPathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root +test.test_posix.TestPosixSpawn.test_close_file +test.test_posix.TestPosixSpawnP.test_close_file +test.test_posixpath.PosixPathTest.test_ismount +test.test_signal.StressTest.test_stress_modifying_handlers +test.test_socket.BasicCANTest.testFilter +test.test_socket.BasicCANTest.testLoopback +test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm +test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCase.test_empty_env +test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCase.test_one_environment_variable +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env +test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable +test.test_venv.BasicTest.test_zippath_from_non_installed_posix \ No newline at end of file From a8cd37407f44d4bd47e893742c76be932a25dd81 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 3 May 2024 11:43:10 -0700 Subject: [PATCH 3/4] remove workspace var and exclude --- .github/workflows/jit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 26284a0a0c9b3a..2c4b56d78f2787 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -93,7 +93,7 @@ jobs: choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No PGO or tests (yet): - name: Emulated Windows @@ -143,4 +143,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --ignorefile=${{ github.workspace }}/Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --ignorefile=./Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From b84747abc3079976691306357b3e23a75a555517 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 3 May 2024 11:44:14 -0700 Subject: [PATCH 4/4] remove ./ --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 2c4b56d78f2787..014cced8e81b52 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -143,4 +143,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --ignorefile=./Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3