Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0df35a9

Browse files
benjaminpbenjaminp
committed
Merged revisions 74841 via svnmerge from
svn+ssh://[email protected]/python/trunk ........ r74841 | thomas.wouters | 2009-09-16 14:55:54 -0500 (Wed, 16 Sep 2009) | 23 lines Fix issue #1590864, multiple threads and fork() can cause deadlocks, by acquiring the import lock around fork() calls. This prevents other threads from having that lock while the fork happens, and is the recommended way of dealing with such issues. There are two other locks we care about, the GIL and the Thread Local Storage lock. The GIL is obviously held when calling Python functions like os.fork(), and the TLS lock is explicitly reallocated instead, while also deleting now-orphaned TLS data. This only fixes calls to os.fork(), not extension modules or embedding programs calling C's fork() directly. Solving that requires a new set of API functions, and possibly a rewrite of the Python/thread_*.c mess. Add a warning explaining the problem to the documentation in the mean time. This also changes behaviour a little on AIX. Before, AIX (but only AIX) was getting the import lock reallocated, seemingly to avoid this very same problem. This is not the right approach, because the import lock is a re-entrant one, and reallocating would do the wrong thing when forking while holding the import lock. Will backport to 2.6, minus the tiny AIX behaviour change. ........
1 parent 60e4cae commit 0df35a9

5 files changed

Lines changed: 108 additions & 22 deletions

File tree

‎Doc/c-api/init.rst‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,22 @@ supports the creation of additional interpreters (using
520520
:cfunc:`Py_NewInterpreter`), but mixing multiple interpreters and the
521521
:cfunc:`PyGILState_\*` API is unsupported.
522522

523+
Another important thing to note about threads is their behaviour in the face
524+
of the C :cfunc:`fork` call. On most systems with :cfunc:`fork`, after a
525+
process forks only the thread that issued the fork will exist. That also
526+
means any locks held by other threads will never be released. Python solves
527+
this for :func:`os.fork` by acquiring the locks it uses internally before
528+
the fork, and releasing them afterwards. In addition, it resets any
529+
:ref:`lock-objects` in the child. When extending or embedding Python, there
530+
is no way to inform Python of additional (non-Python) locks that need to be
531+
acquired before or reset after a fork. OS facilities such as
532+
:cfunc:`posix_atfork` would need to be used to accomplish the same thing.
533+
Additionally, when extending or embedding Python, calling :cfunc:`fork`
534+
directly rather than through :func:`os.fork` (and returning to or calling
535+
into Python) may result in a deadlock by one of Python's internal locks
536+
being held by a thread that is defunct after the fork.
537+
:cfunc:`PyOS_AfterFork` tries to reset the necessary locks, but is not
538+
always able to.
523539

524540
.. ctype:: PyInterpreterState
525541

‎Include/import.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,14 @@ PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m);
2727
PyAPI_FUNC(void) PyImport_Cleanup(void);
2828
PyAPI_FUNC(int) PyImport_ImportFrozenModule(char *);
2929

30+
#ifdef WITH_THREAD
31+
PyAPI_FUNC(void) _PyImport_AcquireLock(void);
32+
PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
33+
#else
34+
#define _PyImport_AcquireLock()
35+
#define _PyImport_ReleaseLock() 1
36+
#endif
37+
3038
PyAPI_FUNC(struct filedescr *) _PyImport_FindModule(
3139
const char *, PyObject *, char *, size_t, FILE **, PyObject **);
3240
PyAPI_FUNC(int) _PyImport_IsScript(struct filedescr *);

‎Lib/test/test_fork1.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
"""This test checks for correct fork() behavior.
22
"""
33

4+
import errno
5+
import imp
46
import os
7+
import signal
8+
import sys
59
import time
10+
import threading
11+
612
from test.fork_wait import ForkWait
713
from test.support import run_unittest, reap_children, get_attribute
814

@@ -23,6 +29,41 @@ def wait_impl(self, cpid):
2329
self.assertEqual(spid, cpid)
2430
self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8))
2531

32+
def test_import_lock_fork(self):
33+
import_started = threading.Event()
34+
fake_module_name = "fake test module"
35+
partial_module = "partial"
36+
complete_module = "complete"
37+
def importer():
38+
imp.acquire_lock()
39+
sys.modules[fake_module_name] = partial_module
40+
import_started.set()
41+
time.sleep(0.01) # Give the other thread time to try and acquire.
42+
sys.modules[fake_module_name] = complete_module
43+
imp.release_lock()
44+
t = threading.Thread(target=importer)
45+
t.start()
46+
import_started.wait()
47+
pid = os.fork()
48+
try:
49+
if not pid:
50+
m = __import__(fake_module_name)
51+
if m == complete_module:
52+
os._exit(0)
53+
else:
54+
os._exit(1)
55+
else:
56+
t.join()
57+
# Exitcode 1 means the child got a partial module (bad.) No
58+
# exitcode (but a hang, which manifests as 'got pid 0')
59+
# means the child deadlocked (also bad.)
60+
self.wait_impl(pid)
61+
finally:
62+
try:
63+
os.kill(pid, signal.SIGKILL)
64+
except OSError:
65+
pass
66+
2667
def test_main():
2768
run_unittest(ForkTest)
2869
reap_children()

‎Modules/posixmodule.c‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3721,11 +3721,21 @@ Return 0 to child process and PID of child to parent process.");
37213721
static PyObject *
37223722
posix_fork1(PyObject *self, PyObject *noargs)
37233723
{
3724-
pid_t pid = fork1();
3724+
pid_t pid;
3725+
int result;
3726+
_PyImport_AcquireLock();
3727+
pid = fork1();
3728+
result = _PyImport_ReleaseLock();
37253729
if (pid == -1)
37263730
return posix_error();
37273731
if (pid == 0)
37283732
PyOS_AfterFork();
3733+
if (result < 0) {
3734+
/* Don't clobber the OSError if the fork failed. */
3735+
PyErr_SetString(PyExc_RuntimeError,
3736+
"not holding the import lock");
3737+
return NULL;
3738+
}
37293739
return PyLong_FromPid(pid);
37303740
}
37313741
#endif
@@ -3740,11 +3750,21 @@ Return 0 to child process and PID of child to parent process.");
37403750
static PyObject *
37413751
posix_fork(PyObject *self, PyObject *noargs)
37423752
{
3743-
pid_t pid = fork();
3753+
pid_t pid;
3754+
int result;
3755+
_PyImport_AcquireLock();
3756+
pid = fork();
3757+
result = _PyImport_ReleaseLock();
37443758
if (pid == -1)
37453759
return posix_error();
37463760
if (pid == 0)
37473761
PyOS_AfterFork();
3762+
if (result < 0) {
3763+
/* Don't clobber the OSError if the fork failed. */
3764+
PyErr_SetString(PyExc_RuntimeError,
3765+
"not holding the import lock");
3766+
return NULL;
3767+
}
37483768
return PyLong_FromPid(pid);
37493769
}
37503770
#endif
@@ -3847,14 +3867,22 @@ To both, return fd of newly opened pseudo-terminal.\n");
38473867
static PyObject *
38483868
posix_forkpty(PyObject *self, PyObject *noargs)
38493869
{
3850-
int master_fd = -1;
3870+
int master_fd = -1, result;
38513871
pid_t pid;
38523872

3873+
_PyImport_AcquireLock();
38533874
pid = forkpty(&master_fd, NULL, NULL, NULL);
3875+
result = _PyImport_ReleaseLock();
38543876
if (pid == -1)
38553877
return posix_error();
38563878
if (pid == 0)
38573879
PyOS_AfterFork();
3880+
if (result < 0) {
3881+
/* Don't clobber the OSError if the fork failed. */
3882+
PyErr_SetString(PyExc_RuntimeError,
3883+
"not holding the import lock");
3884+
return NULL;
3885+
}
38583886
return Py_BuildValue("(Ni)", PyLong_FromPid(pid), master_fd);
38593887
}
38603888
#endif

‎Python/import.c‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,8 @@ static PyThread_type_lock import_lock = 0;
262262
static long import_lock_thread = -1;
263263
static int import_lock_level = 0;
264264

265-
static void
266-
lock_import(void)
265+
void
266+
_PyImport_AcquireLock(void)
267267
{
268268
long me = PyThread_get_thread_ident();
269269
if (me == -1)
@@ -287,8 +287,8 @@ lock_import(void)
287287
import_lock_level = 1;
288288
}
289289

290-
static int
291-
unlock_import(void)
290+
int
291+
_PyImport_ReleaseLock(void)
292292
{
293293
long me = PyThread_get_thread_ident();
294294
if (me == -1 || import_lock == NULL)
@@ -303,23 +303,16 @@ unlock_import(void)
303303
return 1;
304304
}
305305

306-
/* This function is called from PyOS_AfterFork to ensure that newly
307-
created child processes do not share locks with the parent. */
306+
/* This function used to be called from PyOS_AfterFork to ensure that newly
307+
created child processes do not share locks with the parent, but for some
308+
reason only on AIX systems. Instead of re-initializing the lock, we now
309+
acquire the import lock around fork() calls. */
308310

309311
void
310312
_PyImport_ReInitLock(void)
311313
{
312-
#ifdef _AIX
313-
if (import_lock != NULL)
314-
import_lock = PyThread_allocate_lock();
315-
#endif
316314
}
317315

318-
#else
319-
320-
#define lock_import()
321-
#define unlock_import() 0
322-
323316
#endif
324317

325318
static PyObject *
@@ -336,7 +329,7 @@ static PyObject *
336329
imp_acquire_lock(PyObject *self, PyObject *noargs)
337330
{
338331
#ifdef WITH_THREAD
339-
lock_import();
332+
_PyImport_AcquireLock();
340333
#endif
341334
Py_INCREF(Py_None);
342335
return Py_None;
@@ -346,7 +339,7 @@ static PyObject *
346339
imp_release_lock(PyObject *self, PyObject *noargs)
347340
{
348341
#ifdef WITH_THREAD
349-
if (unlock_import() < 0) {
342+
if (_PyImport_ReleaseLock() < 0) {
350343
PyErr_SetString(PyExc_RuntimeError,
351344
"not holding the import lock");
352345
return NULL;
@@ -2201,9 +2194,9 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals,
22012194
PyObject *fromlist, int level)
22022195
{
22032196
PyObject *result;
2204-
lock_import();
2197+
_PyImport_AcquireLock();
22052198
result = import_module_level(name, globals, locals, fromlist, level);
2206-
if (unlock_import() < 0) {
2199+
if (_PyImport_ReleaseLock() < 0) {
22072200
Py_XDECREF(result);
22082201
PyErr_SetString(PyExc_RuntimeError,
22092202
"not holding the import lock");

0 commit comments

Comments
 (0)