From ee55d73360da91fd8c9c75c0796e1596a646112b Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 18 Nov 2022 10:58:46 +0100 Subject: [PATCH 1/3] GH-87804: Fix counter overflow in statvfs on macOS On macOS the statvfs interface returns block counts as 32-bit integers, and that results in bad reporting for larger disks. Therefore reimplement statvfs in terms of statfs, which does use 64-bit integers for block counts. Tested using a sparse filesystem image of 100TB. --- Modules/posixmodule.c | 101 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5968c1940e76f4..c8e32049a7b7ab 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -42,6 +42,12 @@ # define EX_OK EXIT_SUCCESS #endif +#ifdef __APPLE__ + /* Needed for the implementation of os.statvfs */ +# include +# include +#endif + /* On android API level 21, 'AT_EACCESS' is not declared although * HAVE_FACCESSAT is defined. */ #ifdef __ANDROID__ @@ -11382,6 +11388,59 @@ os_WSTOPSIG_impl(PyObject *module, int status) #endif #include +#ifdef __APPLE__ +/* On macOS struct statvfs uses 32-bit integers for block counts, + * resulting in overflow when filesystems are larger tan 4TB. Therefore + * os.statvfs is implemented in terms of statfs(2). + */ + +static PyObject* +_pystatvfs_fromstructstatfs(PyObject *module, struct statfs st) { + PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); + if (v == NULL) + return NULL; + + long flags = 0; + if (st.f_flags & MNT_RDONLY) { + flags |= ST_RDONLY; + } + if (st.f_flags & MNT_NOSUID) { + flags |= ST_NOSUID; + } + + _Static_assert(sizeof(st.f_blocks) == sizeof(long long), "assuming large file"); + + PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_iosize)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_bsize)); + PyStructSequence_SET_ITEM(v, 2, + PyLong_FromLongLong((long long) st.f_blocks)); + PyStructSequence_SET_ITEM(v, 3, + PyLong_FromLongLong((long long) st.f_bfree)); + PyStructSequence_SET_ITEM(v, 4, + PyLong_FromLongLong((long long) st.f_bavail)); + PyStructSequence_SET_ITEM(v, 5, + PyLong_FromLongLong((long long) st.f_files)); + PyStructSequence_SET_ITEM(v, 6, + PyLong_FromLongLong((long long) st.f_ffree)); + PyStructSequence_SET_ITEM(v, 7, + PyLong_FromLongLong((long long) st.f_ffree)); + PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) flags)); + + PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) NAME_MAX)); + PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + + return v; +} + +#else + + + static PyObject* _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { PyObject *StatVFSResultType = get_posix_state(module)->StatVFSResultType; @@ -11433,6 +11492,8 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { return v; } +#endif + /*[clinic input] os.fstatvfs @@ -11450,6 +11511,22 @@ os_fstatvfs_impl(PyObject *module, int fd) { int result; int async_err = 0; +#ifdef __APPLE__ + struct statfs st; + /* On macOS os.fstatvfs is implemented using fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + do { + Py_BEGIN_ALLOW_THREADS + result = fstatfs(fd, &st); + Py_END_ALLOW_THREADS + } while (result != 0 && errno == EINTR && + !(async_err = PyErr_CheckSignals())); + if (result != 0) + return (!async_err) ? posix_error() : NULL; + + return _pystatvfs_fromstructstatfs(module, st); +#else struct statvfs st; do { @@ -11462,6 +11539,7 @@ os_fstatvfs_impl(PyObject *module, int fd) return (!async_err) ? posix_error() : NULL; return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_FSTATVFS) && defined(HAVE_SYS_STATVFS_H) */ @@ -11485,6 +11563,28 @@ os_statvfs_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ { int result; + +#ifdef __APPLE__ + /* On macOS os.statvfs is implemented using statfs(2)/fstatfs(2) because + * the former uses 32-bit values for block counts. + */ + struct statfs st; + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = fstatfs(path->fd, &st); + } + else + result = statfs(path->narrow, &st); + Py_END_ALLOW_THREADS + + if (result) { + return path_error(path); + } + + return _pystatvfs_fromstructstatfs(module, st); + +#else struct statvfs st; Py_BEGIN_ALLOW_THREADS @@ -11502,6 +11602,7 @@ os_statvfs_impl(PyObject *module, path_t *path) } return _pystatvfs_fromstructstatvfs(module, st); +#endif } #endif /* defined(HAVE_STATVFS) && defined(HAVE_SYS_STATVFS_H) */ From 8b0c0e5777407b103df28b4e488df02437a41ad0 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:05:37 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst diff --git a/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst b/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst new file mode 100644 index 00000000000000..e6554d5c9f1e1e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-11-18-10-05-35.gh-issue-87804.rhlDmD.rst @@ -0,0 +1 @@ +On macOS the result of ``os.statvfs`` and ``os.fstatvfs`` now correctly report the size of very large disks, in previous versions the reported number of blocks was wrong for disks with at least 2**32 blocks. From 4eae8186ff7e48ed2a9e0bb5806aa74d021bc701 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 22 Dec 2023 11:38:56 +0100 Subject: [PATCH 3/3] Fix linting issue --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 908f6f23f5c641..4708c4d8706dc1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12826,7 +12826,7 @@ os_fstatvfs_impl(PyObject *module, int fd) int async_err = 0; #ifdef __APPLE__ struct statfs st; - /* On macOS os.fstatvfs is implemented using fstatfs(2) because + /* On macOS os.fstatvfs is implemented using fstatfs(2) because * the former uses 32-bit values for block counts. */ do {