From b14e07108082e6c6fd297b2119e89eea23e900f1 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Tue, 14 Feb 2023 16:56:15 +0000 Subject: [PATCH 1/2] fileutils: handle non-blocking pipe IO on Windows Handle erroring operations on non-blocking pipes by reading the _doserrno code. Limit writes on non-blocking pipes that are too large. --- ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 + Python/fileutils.c | 35 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst new file mode 100644 index 00000000000000..099b2c1c07a665 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst @@ -0,0 +1 @@ +Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Python/fileutils.c b/Python/fileutils.c index 22b2257a56d0ec..88758f3f020856 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1750,7 +1750,15 @@ _Py_read(int fd, void *buf, size_t count) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS + _doserrno = 0; n = read(fd, buf, (int)count); + // read() on a non-blocking empty pipe fails with EINVAL, which is + // mapped from the Windows error code ERROR_NO_DATA. + if (n < 0 && errno == EINVAL) { + if (_doserrno == ERROR_NO_DATA) { + errno = EAGAIN; + } + } #else n = read(fd, buf, count); #endif @@ -1804,6 +1812,7 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) } } } + #endif if (count > _PY_WRITE_MAX) { count = _PY_WRITE_MAX; @@ -1814,7 +1823,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif @@ -1829,7 +1849,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) do { errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif From 3626c05fb1fe4cf8bf785ed4a4533d7ed06f7442 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Tue, 14 Feb 2023 22:32:08 +0000 Subject: [PATCH 2/2] gh-101881: os: support blocking functions on Windows Use the GetNamedPipeHandleState and SetNamedPipeHandleState Win32 API functions to add support for os.get_blocking and os.set_blocking. --- Doc/library/os.rst | 12 +++- Include/internal/pycore_fileutils.h | 4 +- Lib/test/test_os.py | 1 + ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 + Modules/clinic/posixmodule.c.h | 18 +----- Modules/posixmodule.c | 2 - Python/fileutils.c | 58 +++++++++++++++++++ 7 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fb091176767f7a..85924d0e48366b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1091,13 +1091,17 @@ as internal buffering of data. See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. function:: isatty(fd, /) @@ -1565,13 +1569,17 @@ or `the MSDN `_ on Windo See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. data:: SF_NODISKIO SF_MNOWAIT diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ac89c43d569c07..f8e2bf22590888 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -160,11 +160,11 @@ PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, PyAPI_FUNC(int) _Py_dup(int fd); -#ifndef MS_WINDOWS PyAPI_FUNC(int) _Py_get_blocking(int fd); PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#else /* MS_WINDOWS */ + +#ifdef MS_WINDOWS PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 387d2581c06fc6..deea207bfdadd9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4099,6 +4099,7 @@ def test_path_t_converter_and_custom_class(self): @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') @unittest.skipIf(support.is_emscripten, "Cannot unset blocking flag") +@unittest.skipIf(sys.platform == 'win32', 'Windows only supports blocking on pipes') class BlockingTests(unittest.TestCase): def test_blocking(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst new file mode 100644 index 00000000000000..ba58dd4f5cb450 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst @@ -0,0 +1 @@ +Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 5e04507ddd6917..dcd25c28370c93 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10402,8 +10402,6 @@ os_set_handle_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t na #endif /* defined(MS_WINDOWS) */ -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_get_blocking__doc__, "get_blocking($module, fd, /)\n" "--\n" @@ -10439,10 +10437,6 @@ os_get_blocking(PyObject *module, PyObject *arg) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_set_blocking__doc__, "set_blocking($module, fd, blocking, /)\n" "--\n" @@ -10482,8 +10476,6 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - PyDoc_STRVAR(os_DirEntry_is_symlink__doc__, "is_symlink($self, /)\n" "--\n" @@ -11789,14 +11781,6 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_SET_HANDLE_INHERITABLE_METHODDEF) */ -#ifndef OS_GET_BLOCKING_METHODDEF - #define OS_GET_BLOCKING_METHODDEF -#endif /* !defined(OS_GET_BLOCKING_METHODDEF) */ - -#ifndef OS_SET_BLOCKING_METHODDEF - #define OS_SET_BLOCKING_METHODDEF -#endif /* !defined(OS_SET_BLOCKING_METHODDEF) */ - #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ @@ -11812,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a3f76228b549e8ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d9e93473aeadaa..524dc7eb1ccc97 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13930,7 +13930,6 @@ os_set_handle_inheritable_impl(PyObject *module, intptr_t handle, } #endif /* MS_WINDOWS */ -#ifndef MS_WINDOWS /*[clinic input] os.get_blocking -> bool fd: int @@ -13978,7 +13977,6 @@ os_set_blocking_impl(PyObject *module, int fd, int blocking) return NULL; Py_RETURN_NONE; } -#endif /* !MS_WINDOWS */ /*[clinic input] diff --git a/Python/fileutils.c b/Python/fileutils.c index 88758f3f020856..897c2f9f4ea160 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2481,6 +2481,64 @@ _Py_set_blocking(int fd, int blocking) return -1; } #else /* MS_WINDOWS */ +int +_Py_get_blocking(int fd) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + return !(mode & PIPE_NOWAIT); +} + +int +_Py_set_blocking(int fd, int blocking) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + if (success) { + if (blocking) { + mode &= ~PIPE_NOWAIT; + } + else { + mode |= PIPE_NOWAIT; + } + success = SetNamedPipeHandleState(handle, &mode, NULL, NULL); + } + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + return 0; +} + void* _Py_get_osfhandle_noraise(int fd) {