Crash report
What happened?
Bug description:
In the free-threaded build, the os.scandir iterator takes no lock on any path. Calling close() on the iterator concurrently with next() data-races on the dirp (DIR *) handle: close() sets dirp = NULL and calls closedir(dirp),
|
ScandirIterator_closedir(ScandirIterator *iterator) |
|
{ |
|
DIR *dirp = iterator->dirp; |
|
|
|
if (!dirp) |
|
return; |
|
|
|
iterator->dirp = NULL; |
|
Py_BEGIN_ALLOW_THREADS |
|
#ifdef HAVE_FDOPENDIR |
|
if (iterator->path.is_fd) { |
|
rewinddir(dirp); |
|
} |
|
#endif |
|
closedir(dirp); |
while __next__() reads dirp and calls readdir(iterator->dirp):
|
ScandirIterator_iternext(PyObject *op) |
|
{ |
|
ScandirIterator *iterator = ScandirIterator_CAST(op); |
|
struct dirent *direntp; |
|
Py_ssize_t name_len; |
|
int is_dot; |
|
PyObject *entry; |
|
|
|
/* Happens if the iterator is iterated twice, or closed explicitly */ |
|
if (!iterator->dirp) |
|
return NULL; |
|
|
|
while (1) { |
|
errno = 0; |
|
Py_BEGIN_ALLOW_THREADS |
|
direntp = readdir(iterator->dirp); |
__next__() re-reads iterator->dirp for readdir() after its if (!iterator->dirp) check, so a concurrent close() in that window leaves readdir() running on a nulled or freed DIR*. This use-after-free crashes with a SEGV inside readdir.
Reproducer:
import os
import tempfile
from threading import Thread, Barrier
d = tempfile.mkdtemp()
for i in range(200):
open(os.path.join(d, f"f{i}"), "w").close()
it = os.scandir(d)
N_CLOSE, N_NEXT = 4, 8
barrier = Barrier(N_CLOSE + N_NEXT)
def closer():
barrier.wait()
for _ in range(20000):
try:
it.close()
except Exception:
pass
def nexter():
barrier.wait()
for _ in range(20000):
try:
next(it)
except StopIteration:
continue
except Exception:
pass
if __name__ == "__main__":
threads = [Thread(target=closer) for _ in range(N_CLOSE)]
threads += [Thread(target=nexter) for _ in range(N_NEXT)]
for t in threads: t.start()
for t in threads: t.join()TSAN Report:
==================
WARNING: ThreadSanitizer: data race (pid=51315)
Write of size 8 at 0x00010c25e040 by thread T3:
#0 ScandirIterator_close posixmodule.c:16949
#1 method_vectorcall_NOARGS descrobject.c:448
...
#20 thread_run _threadmodule.c:388
#21 pythread_wrapper thread_pthread.h:234
Previous read of size 8 at 0x00010c25e040 by thread T12:
#0 ScandirIterator_iternext posixmodule.c:16907
#1 builtin_next bltinmodule.c:1776
...
#18 thread_run _threadmodule.c:388
#19 pythread_wrapper thread_pthread.h:234
SUMMARY: ThreadSanitizer: data race posixmodule.c:16949 in ScandirIterator_close
==================
The same race also crashes (readdir() on the closedir()'d DIR*):
ThreadSanitizer:DEADLYSIGNAL
ERROR: ThreadSanitizer: SEGV on unknown address 0x000000000040
The signal is caused by a READ memory access.
#0 pthread_mutex_lock (libsystem_pthread)
#2 readdir (libsystem_c)
#4 ScandirIterator_iternext posixmodule.c
SUMMARY: ThreadSanitizer: SEGV in pthread_mutex_lock
WARNING: ThreadSanitizer: data race (pid=3558034)
Write of size 8 at 0x7fffb625e360 by thread T1:
#0 ScandirIterator_closedir /cpython/./Modules/posixmodule.c:16879:20
#1 ScandirIterator_close /cpython/./Modules/posixmodule.c:16949:5 (python3.16t+0x597c68)
#2 method_vectorcall_NOARGS /cpython/Objects/descrobject.c:448:24
#3 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#4 PyObject_Vectorcall /cpython/Objects/call.c:327:12
#5 _Py_VectorCallInstrumentation_StackRefSteal /cpython/Python/ceval.c:768:11
#6 _PyEval_EvalFrameDefault /cpython/Python/generated_cases.c.h:1906:35
...
Previous read of size 8 at 0x7fffb625e360 by thread T12:
#0 ScandirIterator_iternext /cpython/./Modules/posixmodule.c:16907:37
#1 builtin_next /cpython/Python/bltinmodule.c:1776:11
#2 cfunction_vectorcall_FASTCALL /cpython/Objects/methodobject.c:449:24
#3 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#4 PyObject_Vectorcall /cpython/Objects/call.c:327:12
#5 _Py_VectorCallInstrumentation_StackRefSteal /cpython/Python/ceval.c:768:11
#6 _PyEval_EvalFrameDefault /cpython/Python/generated_cases.c.h:1906:35
...
SUMMARY: ThreadSanitizer: data race t/cpython/./Modules/posixmodule.c:16879:20 in ScandirIterator_closedir
==================
ThreadSanitizer:DEADLYSIGNAL
==3558533==ERROR: ThreadSanitizer: SEGV on unknown address 0x000000000004 (pc 0x7ffff7ced99e bp 0x7fffab5f87d0 sp 0x7fffab5f87a0 T3558539)
==3558533==The signal is caused by a WRITE memory access.
==3558533==Hint: address points to the zero page.
#0 readdir dirent/../sysdeps/unix/sysv/linux/readdir64.c:37:3
#1 readdir64 <null>
#2 ScandirIterator_iternext /cpython/./Modules/posixmodule.c:16907:19
#3 builtin_next /cpython/Python/bltinmodule.c:1776:11
#4 _Py_BuiltinCallFast_StackRef /cpython/Python/ceval.c:817:11
#5 _PyEval_EvalFrameDefault /cpython/Python/generated_cases.c.h:2510:35
#6 _PyEval_EvalFrame /cpython/./Include/internal/pycore_ceval.h:122:16
#7 _PyEval_Vector /cpython/Python/ceval.c:2141:12
#8 _PyFunction_Vectorcall /cpython/Objects/call.c
#9 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#10 _PyObject_VectorcallPrepend /cpython/Objects/call.c:855:20
#11 method_vectorcall /cpython/Objects/classobject.c:55:12
#12 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#13 context_run /cpython/Python/context.c:731:29
#14 method_vectorcall_FASTCALL_KEYWORDS /cpython/Objects/descrobject.c:421:24
#15 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#16 PyObject_Vectorcall /cpython/Objects/call.c:327:12
#17 _Py_VectorCallInstrumentation_StackRefSteal /cpython/Python/ceval.c:768:11
#18 _PyEval_EvalFrameDefault /cpython/Python/generated_cases.c.h:1906:35
#19 _PyEval_EvalFrame /cpython/./Include/internal/pycore_ceval.h:122:16
#20 _PyEval_Vector /cpython/Python/ceval.c:2141:12
#21 _PyFunction_Vectorcall /cpython/Objects/call.c
#22 _PyObject_VectorcallTstate /cpython/./Include/internal/pycore_call.h:144:11
#23 _PyObject_VectorcallPrepend /cpython/Objects/call.c:855:20
#24 method_vectorcall /cpython/Objects/classobject.c:55:12
#25 _PyVectorcall_Call /cpython/Objects/call.c:273:16
#26 _PyObject_Call /cpython/Objects/call.c:348:16
#27 PyObject_Call /cpython/Objects/call.c:373:12
#28 thread_run /cpython/./Modules/_threadmodule.c:388:21
#29 pythread_wrapper /cpython/Python/thread_pthread.h:234:5
#30 __tsan_thread_start_func <null>
#31 start_thread nptl/pthread_create.c:447:8
#32 clone3 misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
ThreadSanitizer can not provide additional info.
SUMMARY: ThreadSanitizer: SEGV dirent/../sysdeps/unix/sysv/linux/readdir64.c:37:3 in readdir
==3558533==ABORTING
CPython versions tested on:
CPython main branch
Operating systems tested on:
Linux
Output from running 'python -VV' on the command line:
No response
Linked PRs
Crash report
What happened?
Bug description:
In the free-threaded build, the
os.scandiriterator takes no lock on any path. Callingclose()on the iterator concurrently withnext()data-races on thedirp(DIR *) handle:close()setsdirp = NULLand callsclosedir(dirp),cpython/Modules/posixmodule.c
Lines 16872 to 16886 in 9e863fa
while
__next__()readsdirpand callsreaddir(iterator->dirp):cpython/Modules/posixmodule.c
Lines 16892 to 16907 in 9e863fa
__next__()re-readsiterator->dirpforreaddir()after itsif (!iterator->dirp)check, so a concurrentclose()in that window leavesreaddir()running on a nulled or freedDIR*. This use-after-free crashes with a SEGV insidereaddir.Reproducer:
TSAN Report:
The same race also crashes (
readdir()on theclosedir()'dDIR*):CPython versions tested on:
CPython main branch
Operating systems tested on:
Linux
Output from running 'python -VV' on the command line:
No response
Linked PRs