Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[3.7] bpo-24658: Fix read/write greater than 2 GiB on macOS (GH-1705) #9936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 18, 2018
Merged

[3.7] bpo-24658: Fix read/write greater than 2 GiB on macOS (GH-1705) #9936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions Include/fileutils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,19 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx(
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);

#if defined(MS_WINDOWS) || defined(__APPLE__)
/* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
On macOS 10.13, read() and write() with more than INT_MAX bytes
fail with EINVAL (bpo-24658). */
# define _PY_READ_MAX INT_MAX
# define _PY_WRITE_MAX INT_MAX
#else
/* write() should truncate the input to PY_SSIZE_T_MAX bytes,
but it's safer to do it ourself to have a portable behaviour */
# define _PY_READ_MAX PY_SSIZE_T_MAX
# define _PY_WRITE_MAX PY_SSIZE_T_MAX
#endif

#ifdef MS_WINDOWS
struct _Py_stat_struct {
unsigned long st_dev;
Expand Down
13 changes: 11 additions & 2 deletions Lib/test/test_largefile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
import stat
import sys
import unittest
from test.support import TESTFN, requires, unlink
from test.support import TESTFN, requires, unlink, bigmemtest
import io # C implementation of io
import _pyio as pyio # Python implementation of io

# size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
size = 2500000000
size = 2_500_000_000

class LargeFileTest:
"""Test that each file function works as expected for large
Expand Down Expand Up @@ -45,6 +45,15 @@ def tearDownClass(cls):
raise cls.failureException('File was not truncated by opening '
'with mode "wb"')

# _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
# so memuse=2 is needed
@bigmemtest(size=size, memuse=2, dry_run=False)
def test_large_read(self, _size):
# bpo-24658: Test that a read greater than 2GB does not fail.
with self.open(TESTFN, "rb") as f:
self.assertEqual(len(f.read()), size + 1)
self.assertEqual(f.tell(), size + 1)

def test_osstat(self):
self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)

Expand Down
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
8 changes: 3 additions & 5 deletions Modules/_io/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -791,11 +791,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size)
if (size < 0)
return _io_FileIO_readall_impl(self);

#ifdef MS_WINDOWS
/* On Windows, the count parameter of read() is an int */
if (size > INT_MAX)
size = INT_MAX;
#endif
if (size > _PY_READ_MAX) {
size = _PY_READ_MAX;
}

bytes = PyBytes_FromStringAndSize(NULL, size);
if (bytes == NULL)
Expand Down
24 changes: 5 additions & 19 deletions Python/fileutils.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1363,18 +1363,9 @@ _Py_read(int fd, void *buf, size_t count)
* handler raised an exception. */
assert(!PyErr_Occurred());

#ifdef MS_WINDOWS
if (count > INT_MAX) {
/* On Windows, the count parameter of read() is an int */
count = INT_MAX;
}
#else
if (count > PY_SSIZE_T_MAX) {
/* if count is greater than PY_SSIZE_T_MAX,
* read() result is undefined */
count = PY_SSIZE_T_MAX;
if (count > _PY_READ_MAX) {
count = _PY_READ_MAX;
}
#endif

_Py_BEGIN_SUPPRESS_IPH
do {
Expand Down Expand Up @@ -1425,15 +1416,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
depending on heap usage). */
count = 32767;
}
else if (count > INT_MAX)
count = INT_MAX;
#else
if (count > PY_SSIZE_T_MAX) {
/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer
* to do it ourself to have a portable behaviour. */
count = PY_SSIZE_T_MAX;
}
#endif
if (count > _PY_WRITE_MAX) {
count = _PY_WRITE_MAX;
}

if (gil_held) {
do {
Expand Down