Fix multiprocessing warnings when runnign tests on Python 3.12 (#16564)

JukkaL · web-flow · commit 379d59e1a7e1 · 2023-11-27T16:41:36.000Z
I saw a bunch of warnings when running tests in parallel using pytest. When running tests sequentially using `-n0` I didn't see warnings. This only seems to happen on Linux. The warnings were like these, which can be fixed by avoiding the use of fork, and using forkserver instead: ``` mypy/test/teststubgen.py::StubgenPythonSuite::stubgen.test::testNestedClass_inspect /usr/local/lib/python3.12/multiprocessing/popen_fork.py:66: DeprecationWarning: This process (pid=84587) is multi-threaded, use of fork() may lead to deadlocks in the child. self.pid = os.fork() ``` Relevant discussion: https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555
diff --git a/mypy/moduleinspect.py b/mypy/moduleinspect.py
@@ -8,7 +8,7 @@
 import pkgutil
 import queue
 import sys
-from multiprocessing import Process, Queue
+from multiprocessing import Queue, get_context
 from types import ModuleType
 
 
@@ -123,9 +123,13 @@ def __init__(self) -> None:
         self._start()
 
     def _start(self) -> None:
-        self.tasks: Queue[str] = Queue()
-        self.results: Queue[ModuleProperties | str] = Queue()
-        self.proc = Process(target=worker, args=(self.tasks, self.results, sys.path))
+        if sys.platform == "linux":
+            ctx = get_context("forkserver")
+        else:
+            ctx = get_context("spawn")
+        self.tasks: Queue[str] = ctx.Queue()
+        self.results: Queue[ModuleProperties | str] = ctx.Queue()
+        self.proc = ctx.Process(target=worker, args=(self.tasks, self.results, sys.path))
         self.proc.start()
         self.counter = 0  # Number of successful roundtrips
 
diff --git a/mypy/test/testipc.py b/mypy/test/testipc.py
@@ -2,7 +2,7 @@
 
 import sys
 import time
-from multiprocessing import Process, Queue
+from multiprocessing import Queue, get_context
 from unittest import TestCase, main
 
 import pytest
@@ -35,10 +35,17 @@ def server_multi_message_echo(q: Queue[str]) -> None:
 
 
 class IPCTests(TestCase):
+    def setUp(self) -> None:
+        if sys.platform == "linux":
+            # The default "fork" start method is potentially unsafe
+            self.ctx = get_context("forkserver")
+        else:
+            self.ctx = get_context("spawn")
+
     def test_transaction_large(self) -> None:
-        queue: Queue[str] = Queue()
+        queue: Queue[str] = self.ctx.Queue()
         msg = "t" * 200000  # longer than the max read size of 100_000
-        p = Process(target=server, args=(msg, queue), daemon=True)
+        p = self.ctx.Process(target=server, args=(msg, queue), daemon=True)
         p.start()
         connection_name = queue.get()
         with IPCClient(connection_name, timeout=1) as client:
@@ -49,9 +56,9 @@ def test_transaction_large(self) -> None:
         p.join()
 
     def test_connect_twice(self) -> None:
-        queue: Queue[str] = Queue()
+        queue: Queue[str] = self.ctx.Queue()
         msg = "this is a test message"
-        p = Process(target=server, args=(msg, queue), daemon=True)
+        p = self.ctx.Process(target=server, args=(msg, queue), daemon=True)
         p.start()
         connection_name = queue.get()
         with IPCClient(connection_name, timeout=1) as client:
@@ -67,8 +74,8 @@ def test_connect_twice(self) -> None:
         assert p.exitcode == 0
 
     def test_multiple_messages(self) -> None:
-        queue: Queue[str] = Queue()
-        p = Process(target=server_multi_message_echo, args=(queue,), daemon=True)
+        queue: Queue[str] = self.ctx.Queue()
+        p = self.ctx.Process(target=server_multi_message_echo, args=(queue,), daemon=True)
         p.start()
         connection_name = queue.get()
         with IPCClient(connection_name, timeout=1) as client:
