From 34cd3729a95bfd9f9de689db9d19f66a3369e4ce Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 01:34:46 +0900 Subject: [PATCH 01/28] gh-112536: Setting up TSAN test at Github Action --- .github/workflows/build.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d43b83e830e1fb..a86ae51ab497ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -484,6 +484,31 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan_free_threading: + name: 'Thread sanitizer (free-threading)' + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for TSAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 + - name: TSAN Option Setup + run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + - name: Configure CPython + run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz @@ -542,6 +567,7 @@ jobs: - build_windows_free_threading - test_hypothesis - build_asan + - build_tsan_free_threading - cifuzz runs-on: ubuntu-latest @@ -575,6 +601,7 @@ jobs: build_windows, build_windows_free_threading, build_asan, + build_tsan_free_threading, ' || '' }} From 8f789dfb1b8fb10a958b9fd25913f512f2415db7 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:27:56 +0900 Subject: [PATCH 02/28] Apply TSAN to the default build too --- .github/workflows/build.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a86ae51ab497ff..c5cdc496388535 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -484,6 +484,27 @@ jobs: - name: Tests run: xvfb-run make test + build_tsan: + name: 'Thread sanitizer' + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: TSAN Option Setup + run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + - name: Configure CPython + run: ./configure --with-thread-sanitizer --without-pymalloc + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' runs-on: ubuntu-20.04 @@ -494,14 +515,10 @@ jobs: - uses: actions/checkout@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for TSAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 + run: ./configure --disable-gil --with-thread-sanitizer - name: Build CPython run: make -j4 - name: Display build info @@ -567,6 +584,7 @@ jobs: - build_windows_free_threading - test_hypothesis - build_asan + - build_tsan - build_tsan_free_threading - cifuzz @@ -601,6 +619,7 @@ jobs: build_windows, build_windows_free_threading, build_asan, + build_tsan, build_tsan_free_threading, ' || '' From 432c3a2d750304cdf00ad46302d254bf184847bc Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:37:18 +0900 Subject: [PATCH 03/28] Update --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5cdc496388535..40b4d31dad9dbc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -497,7 +497,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc + run: ./configure --with-thread-sanitizer --without-pymalloc --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info @@ -518,7 +518,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer + run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info From 7c04959ec5315d3fbe277e9c17b7a0cbaa9c51af Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:42:20 +0900 Subject: [PATCH 04/28] Use gcc 11 to resolve compiler issue --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 40b4d31dad9dbc..bd6f7f01b77675 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,6 +494,10 @@ jobs: - uses: actions/checkout@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-11 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 11 - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython @@ -515,6 +519,10 @@ jobs: - uses: actions/checkout@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-11 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 11 - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython From 4d271cb5ce2fa64a954ba9a5a9320959ca81baa2 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:52:32 +0900 Subject: [PATCH 05/28] Remove test_logging from the TSAN test for the temporary --- Lib/test/libregrtest/tsan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 150fcec8816179..8231d6443a7275 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -9,7 +9,6 @@ 'test_imaplib', 'test_importlib', 'test_io', - 'test_logging', 'test_ssl', 'test_syslog', 'test_thread', From 853ea8c34db01f99c85d35adcd037cd891ff0bab Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:57:11 +0900 Subject: [PATCH 06/28] Test --with-pydebug --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd6f7f01b77675..d16b8b6983cff9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -501,7 +501,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc --disable-ipv6 + run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug - name: Build CPython run: make -j4 - name: Display build info @@ -526,7 +526,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 + run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug - name: Build CPython run: make -j4 - name: Display build info From 43135d48c5793873880d135df60f63385d63b920 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 02:58:23 +0900 Subject: [PATCH 07/28] Use Ubuntu 22.04 --- .github/workflows/build.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d16b8b6983cff9..a92f88d81b542b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -486,7 +486,7 @@ jobs: build_tsan: name: 'Thread sanitizer' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -494,10 +494,6 @@ jobs: - uses: actions/checkout@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-11 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 11 - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython @@ -511,7 +507,7 @@ jobs: build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -519,10 +515,6 @@ jobs: - uses: actions/checkout@v4 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-11 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 11 - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython From 7304fcacfb243e4d4f3cdbcc2d8a10baceac1e28 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:03:47 +0900 Subject: [PATCH 08/28] Use --disable-ipv6 sigh.. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92f88d81b542b..55a48e218d125a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -497,7 +497,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug + run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info @@ -518,7 +518,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug + run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info From 37d048d7afe0a4e61f371b254624b6d4ad82f1e5 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:08:48 +0900 Subject: [PATCH 09/28] revert --with-pydebug --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55a48e218d125a..e316da1b4a3e52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -497,7 +497,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug --disable-ipv6 + run: ./configure --with-thread-sanitizer --without-pymalloc --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info @@ -518,7 +518,7 @@ jobs: - name: TSAN Option Setup run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug --disable-ipv6 + run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 - name: Build CPython run: make -j4 - name: Display build info From b05c9f72a945c69808033899a144d3a9c9ab1ce9 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:11:49 +0900 Subject: [PATCH 10/28] Use clang --- .github/workflows/build.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e316da1b4a3e52..0414fee0801893 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,9 +493,14 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt install -y clang - name: TSAN Option Setup - run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + run: | + echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + echo "CC=$(which clang)" >> $GITHUB_ENV + echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython run: ./configure --with-thread-sanitizer --without-pymalloc --disable-ipv6 - name: Build CPython @@ -514,9 +519,14 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt install -y clang - name: TSAN Option Setup - run: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + run: | + echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + echo "CC=$(which clang)" >> $GITHUB_ENV + echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 - name: Build CPython From 7e373ee20783ff9fee59351ba38c6f8ecdb2c79b Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:22:07 +0900 Subject: [PATCH 11/28] Reduce ASLR to avoid TSAN crashing --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0414fee0801893..d2fc9e5cc26450 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -496,6 +496,8 @@ jobs: run: | sudo ./.github/workflows/posix-deps-apt.sh sudo apt install -y clang + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV @@ -522,6 +524,8 @@ jobs: run: | sudo ./.github/workflows/posix-deps-apt.sh sudo apt install -y clang + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV From 920d8b4bba89f7c8cf6eadb74344092491972521 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:28:31 +0900 Subject: [PATCH 12/28] Rebirth test_logging --- .github/workflows/build.yml | 4 ++-- Lib/test/libregrtest/tsan.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2fc9e5cc26450..d7cd1b2131f359 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -504,7 +504,7 @@ jobs: echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc --disable-ipv6 + run: ./configure --with-thread-sanitizer --without-pymalloc - name: Build CPython run: make -j4 - name: Display build info @@ -532,7 +532,7 @@ jobs: echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --disable-ipv6 + run: ./configure --disable-gil --with-thread-sanitizer - name: Build CPython run: make -j4 - name: Display build info diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 8231d6443a7275..150fcec8816179 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -9,6 +9,7 @@ 'test_imaplib', 'test_importlib', 'test_io', + 'test_logging', 'test_ssl', 'test_syslog', 'test_thread', From 73cbf74544b8143ce6dcb72450a65a2e6f6c3339 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:34:45 +0900 Subject: [PATCH 13/28] Add --with-pydebug --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d7cd1b2131f359..fa25c52e2f89b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -504,7 +504,7 @@ jobs: echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc + run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug - name: Build CPython run: make -j4 - name: Display build info @@ -532,7 +532,7 @@ jobs: echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer + run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug - name: Build CPython run: make -j4 - name: Display build info From 0ff82128c90a647ce78fba98ac497013ed92e614 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:42:10 +0900 Subject: [PATCH 14/28] Update Tools/tsan/supressions.txt --- Tools/tsan/supressions.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt index 448dfac8005c79..617f9438272fd6 100644 --- a/Tools/tsan/supressions.txt +++ b/Tools/tsan/supressions.txt @@ -3,3 +3,5 @@ race:get_allocator_unlocked race:set_allocator_unlocked race:mi_heap_visit_pages race:_mi_heap_delayed_free_partial + +thread:pthread_create From c6dfb25f25aed985e39b0491c7fc87e85acc5149 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 03:45:32 +0900 Subject: [PATCH 15/28] Add ccache --- .github/workflows/build.yml | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa25c52e2f89b2..745303f10b49de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -492,6 +492,13 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Install Dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh @@ -503,8 +510,16 @@ jobs: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Configure CPython - run: ./configure --with-thread-sanitizer --without-pymalloc --with-pydebug + run: ./configure --config-cache --with-thread-sanitizer --without-pymalloc --with-pydebug - name: Build CPython run: make -j4 - name: Display build info @@ -520,6 +535,13 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Install Dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh @@ -531,8 +553,16 @@ jobs: echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV echo "CC=$(which clang)" >> $GITHUB_ENV echo "CXX=$(which clang++)" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Configure CPython - run: ./configure --disable-gil --with-thread-sanitizer --with-pydebug + run: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug - name: Build CPython run: make -j4 - name: Display build info From e2e90eefdb70920bbcecd663ca080bb06938e21f Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:14:48 +0900 Subject: [PATCH 16/28] Address code review --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 745303f10b49de..1c107218b49f80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -508,8 +508,8 @@ jobs: - name: TSAN Option Setup run: | echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - echo "CC=$(which clang)" >> $GITHUB_ENV - echo "CXX=$(which clang++)" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV @@ -519,7 +519,7 @@ jobs: save: ${{ github.event_name == 'push' }} max-size: "200M" - name: Configure CPython - run: ./configure --config-cache --with-thread-sanitizer --without-pymalloc --with-pydebug + run: ./configure --config-cache --with-thread-sanitizer --with-pydebug - name: Build CPython run: make -j4 - name: Display build info @@ -551,8 +551,8 @@ jobs: - name: TSAN Option Setup run: | echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - echo "CC=$(which clang)" >> $GITHUB_ENV - echo "CXX=$(which clang++)" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV From 2d29262ab65ecd65a55a59d855654ae4a85e0d86 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:25:45 +0900 Subject: [PATCH 17/28] Don't use sem_clockwait with TSAN --- Python/thread_pthread.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 64cc60053e6cf7..65d366e91c322a 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -95,6 +95,10 @@ #endif #endif +/* Thread sanitizer doesn't currently support sem_clockwait */ +#ifdef _Py_THREAD_SANITIZER +#undef HAVE_SEM_CLOCKWAIT +#endif /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: From e2265a52e61ab13aebfe5086244754d88047682f Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:26:00 +0900 Subject: [PATCH 18/28] Skip tests from test_threading and test_concurrent_futures --- Lib/test/test_concurrent_futures/util.py | 4 ++++ Lib/test/test_threading.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 3e855031913042..3b8ec3e205d5aa 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -85,6 +85,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,6 +113,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b5c37c948c8c3..9ba194b186a204 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -50,6 +50,12 @@ def skip_unless_reliable_fork(test): return test +def skip_for_tsan(test): + if support.check_sanitizer(thread=True): + return test.skip("TSAN doesn't support threads after fork") + return test + + def requires_subinterpreters(meth): """Decorator to skip a test if subinterpreters are not supported.""" return unittest.skipIf(interpreters is None, @@ -634,6 +640,7 @@ def test_daemon_param(self): self.assertTrue(t.daemon) @skip_unless_reliable_fork + @skip_for_tsan def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -703,6 +710,7 @@ def f(): @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + @skip_for_tsan def test_main_thread_after_fork(self): code = """if 1: import os, threading @@ -1271,6 +1279,7 @@ def test_2_join_in_forked_process(self): self._run_and_join(script) @skip_unless_reliable_fork + @skip_for_tsan def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. From 414c0e942ff46d49ba92ee86cb4bc418cf40bf2f Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:28:52 +0900 Subject: [PATCH 19/28] Revert Tools/tsan/supressions.txt changes --- Tools/tsan/supressions.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt index 617f9438272fd6..448dfac8005c79 100644 --- a/Tools/tsan/supressions.txt +++ b/Tools/tsan/supressions.txt @@ -3,5 +3,3 @@ race:get_allocator_unlocked race:set_allocator_unlocked race:mi_heap_visit_pages race:_mi_heap_delayed_free_partial - -thread:pthread_create From 71dc4aa260929ba00046a25fc28e21042c3f0132 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:36:13 +0900 Subject: [PATCH 20/28] Add skip_if_tsan --- Lib/test/test_logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 32bb5171a3f757..c84eca51b52362 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -80,6 +80,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -731,6 +734,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): From f399bd2724e20154599df225b22bfc760c976ca9 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:38:28 +0900 Subject: [PATCH 21/28] nit --- Lib/test/test_threading.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 9ba194b186a204..9769cb41e3e689 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -50,10 +50,9 @@ def skip_unless_reliable_fork(test): return test -def skip_for_tsan(test): - if support.check_sanitizer(thread=True): - return test.skip("TSAN doesn't support threads after fork") - return test +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") def requires_subinterpreters(meth): @@ -640,7 +639,7 @@ def test_daemon_param(self): self.assertTrue(t.daemon) @skip_unless_reliable_fork - @skip_for_tsan + @skip_if_tsan_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. @@ -710,7 +709,7 @@ def f(): @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") - @skip_for_tsan + @skip_if_tsan_fork def test_main_thread_after_fork(self): code = """if 1: import os, threading @@ -1279,7 +1278,7 @@ def test_2_join_in_forked_process(self): self._run_and_join(script) @skip_unless_reliable_fork - @skip_for_tsan + @skip_if_tsan_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. From ec743ff46008c974bb634c0dccf36908fd6e7699 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:53:48 +0900 Subject: [PATCH 22/28] Use reusable workflow --- .github/workflows/build.yml | 89 ++++++---------------------- .github/workflows/reusable-tsan.yaml | 53 +++++++++++++++++ 2 files changed, 70 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/reusable-tsan.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c107218b49f80..70dbdfa7c45b96 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -490,42 +490,14 @@ jobs: timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' - steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - sudo apt install -y clang - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN Option Setup - run: | - echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-thread-sanitizer --with-pydebug - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-thread-sanitizer \ + --with-pydebug build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' @@ -533,42 +505,15 @@ jobs: timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' - steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - sudo apt install -y clang - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN Option Setup - run: | - echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --disable-gil \ + --with-thread-sanitizer \ + --with-pydebug4 # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: diff --git a/.github/workflows/reusable-tsan.yaml b/.github/workflows/reusable-tsan.yaml new file mode 100644 index 00000000000000..c64759dffeb7fd --- /dev/null +++ b/.github/workflows/reusable-tsan.yaml @@ -0,0 +1,53 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt install -y clang + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ${{ inputs.options }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 From e99087f783f76e6a26f4addbfe7c3caa6d3ba5a2 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 10:56:37 +0900 Subject: [PATCH 23/28] nit --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70dbdfa7c45b96..fb0e4c18f87325 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -513,7 +513,7 @@ jobs: --config-cache \ --disable-gil \ --with-thread-sanitizer \ - --with-pydebug4 + --with-pydebug # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: From 8bfee0195b8a338fca69a765cd7390c7b06f940b Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 11:05:29 +0900 Subject: [PATCH 24/28] Rename to .github/workflows/reusable-tsan.yml --- .github/workflows/{reusable-tsan.yaml => reusable-tsan.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{reusable-tsan.yaml => reusable-tsan.yml} (100%) diff --git a/.github/workflows/reusable-tsan.yaml b/.github/workflows/reusable-tsan.yml similarity index 100% rename from .github/workflows/reusable-tsan.yaml rename to .github/workflows/reusable-tsan.yml From 8577792fcab64fa07d0c8c9bc557e35080f10053 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 11:17:44 +0900 Subject: [PATCH 25/28] fix --- .github/workflows/build.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb0e4c18f87325..b0208e0542bb3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -486,8 +486,6 @@ jobs: build_tsan: name: 'Thread sanitizer' - runs-on: ubuntu-22.04 - timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-tsan.yml @@ -501,8 +499,6 @@ jobs: build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' - runs-on: ubuntu-22.04 - timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' uses: ./.github/workflows/reusable-tsan.yml From f9c6e56488c43c85ec60e32a57dac5a6c55361c1 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 11:19:54 +0900 Subject: [PATCH 26/28] fix --- .github/workflows/reusable-tsan.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index c64759dffeb7fd..96a9c1b0cda3c3 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -13,8 +13,6 @@ jobs: name: 'Thread sanitizer' runs-on: ubuntu-22.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 - name: Runner image version From d9afff0bc698595bc8645bc9587901a9d425af88 Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 11:26:10 +0900 Subject: [PATCH 27/28] Fix --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0208e0542bb3a..3873aaff6b92fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -492,7 +492,7 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} options: | - ../cpython-ro-srcdir/configure \ + ./configure \ --config-cache \ --with-thread-sanitizer \ --with-pydebug @@ -505,7 +505,7 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} options: | - ../cpython-ro-srcdir/configure \ + ./configure \ --config-cache \ --disable-gil \ --with-thread-sanitizer \ From e88086f54d3a69ec74ada9940d52e6f1cc4d227a Mon Sep 17 00:00:00 2001 From: Donghee Nals Date: Sat, 16 Mar 2024 11:27:14 +0900 Subject: [PATCH 28/28] nit --- .github/workflows/build.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3873aaff6b92fc..e36859e728b67f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -491,11 +491,7 @@ jobs: uses: ./.github/workflows/reusable-tsan.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ./configure \ - --config-cache \ - --with-thread-sanitizer \ - --with-pydebug + options: ./configure --config-cache --with-thread-sanitizer --with-pydebug build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' @@ -504,12 +500,7 @@ jobs: uses: ./.github/workflows/reusable-tsan.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ./configure \ - --config-cache \ - --disable-gil \ - --with-thread-sanitizer \ - --with-pydebug + options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: