From f8101293e55069c9841e3a2ab3ed0d5f6a8dc3c3 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Sun, 21 May 2017 23:12:22 +0200 Subject: [PATCH 01/27] bpo-24658: Fix read/write on file with a size greater than 2GB on OSX bpo-24658: Fix read/write on file with a size greater than 2GB on OSX --- Include/fileutils.h | 11 +++++++++++ Lib/test/test_largefile.py | 15 ++++++++++++++- Modules/_io/fileio.c | 8 +++----- Python/fileutils.c | 24 +++++------------------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index f0a8e2c61a4fe0..ebc3d916c7c364 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -81,6 +81,17 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _Py_device_encoding(int); +#if defined(MS_WINDOWS) || defined(__APPLE__) +/* On Windows, the count parameter of read() is an int + See issue #24658 +*/ +#define _PY_READ_MAX INT_MAX +#define _PY_WRITE_MAX INT_MAX +#else +#define _PY_READ_MAX PY_SSIZE_T_MAX +#define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 21296ccafe541f..bc2f69122f0907 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -5,7 +5,7 @@ import stat import sys import unittest -from test.support import TESTFN, requires, unlink +from test.support import TESTFN, requires, unlink, bigmemtest, _2G import io # C implementation of io import _pyio as pyio # Python implementation of io @@ -45,6 +45,19 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') + def test_large_reads_writes(self): + # see issue #24658 + requires('largefile', + 'test requires %s bytes and a long time to run' % size) + with self.open(TESTFN, "wb") as f: + b = b'x' * size + self.assertEqual(f.write(b), size) + self.assertEqual(f.tell(), size) + + with self.open(TESTFN, "rb") as f: + f.read() + self.assertEqual(f.tell(), size) + def test_osstat(self): self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index c0e43e0ae41a8f..44d51c9faf5d1f 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -791,11 +791,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size) if (size < 0) return _io_FileIO_readall_impl(self); -#ifdef MS_WINDOWS - /* On Windows, the count parameter of read() is an int */ - if (size > INT_MAX) - size = INT_MAX; -#endif + if (size > _PY_READ_MAX) { + size = _PY_READ_MAX; + } bytes = PyBytes_FromStringAndSize(NULL, size); if (bytes == NULL) diff --git a/Python/fileutils.c b/Python/fileutils.c index 0486f865924a49..2c4061e2692c0f 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1471,18 +1471,9 @@ _Py_read(int fd, void *buf, size_t count) * handler raised an exception. */ assert(!PyErr_Occurred()); -#ifdef MS_WINDOWS - if (count > INT_MAX) { - /* On Windows, the count parameter of read() is an int */ - count = INT_MAX; - } -#else - if (count > PY_SSIZE_T_MAX) { - /* if count is greater than PY_SSIZE_T_MAX, - * read() result is undefined */ - count = PY_SSIZE_T_MAX; + if (count > _PY_READ_MAX) { + count = _PY_READ_MAX; } -#endif _Py_BEGIN_SUPPRESS_IPH do { @@ -1533,15 +1524,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) depending on heap usage). */ count = 32767; } - else if (count > INT_MAX) - count = INT_MAX; -#else - if (count > PY_SSIZE_T_MAX) { - /* write() should truncate count to PY_SSIZE_T_MAX, but it's safer - * to do it ourself to have a portable behaviour. */ - count = PY_SSIZE_T_MAX; - } #endif + if (count > _PY_WRITE_MAX) { + count = _PY_WRITE_MAX; + } if (gil_held) { do { From 1b7d4128a50770488c1755e3330ca3d6910ccce0 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 22 May 2017 19:10:36 +0200 Subject: [PATCH 02/27] bpo-24658: Clean the test file for test_large_reads_writes --- Lib/test/test_largefile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index bc2f69122f0907..35fe2d971b8d27 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -49,6 +49,7 @@ def test_large_reads_writes(self): # see issue #24658 requires('largefile', 'test requires %s bytes and a long time to run' % size) + self.addCleanup(unlink, TESTFN) with self.open(TESTFN, "wb") as f: b = b'x' * size self.assertEqual(f.write(b), size) From 3c447d865a6906ad5a924d6fc8f4a526cd8e26a8 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 22 May 2017 19:11:54 +0200 Subject: [PATCH 03/27] bpo-24658: Improve the comment for the support of OSX for _PY_READ_MAX and _PY_WRITE_MAX --- Include/fileutils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index ebc3d916c7c364..5f84c8b5629190 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -83,7 +83,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) /* On Windows, the count parameter of read() is an int - See issue #24658 + Add the support of MacOS with the issue #24658 */ #define _PY_READ_MAX INT_MAX #define _PY_WRITE_MAX INT_MAX From b5b2754bf1f22448ff52c3fe4741669650746ae6 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 22 May 2017 20:16:02 +0200 Subject: [PATCH 04/27] bpo-24658: Improve the comment about macOS --- Include/fileutils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 5f84c8b5629190..9b0f93e2535a22 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -82,8 +82,8 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) -/* On Windows, the count parameter of read() is an int - Add the support of MacOS with the issue #24658 +/* On Windows, the count parameter of read() is an int. + macOS fails when reading or writing more than 2GB, see #24658 */ #define _PY_READ_MAX INT_MAX #define _PY_WRITE_MAX INT_MAX From 1130c4402c788b0e5d2b3a2f5f24edfcffa6d7ce Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 22 May 2017 21:30:11 +0200 Subject: [PATCH 05/27] bpo-24658: Add the bigmemtest decorator for the test --- Lib/test/test_largefile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 35fe2d971b8d27..634f8b0698f71a 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,7 +45,8 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - def test_large_reads_writes(self): + @bigmemtest(size=_2G, memuse=1) + def test_large_reads_writes(self, _size): # see issue #24658 requires('largefile', 'test requires %s bytes and a long time to run' % size) From 207f00ffb63d554ec4407caaf0b6faf64087cef8 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 00:22:23 +0200 Subject: [PATCH 06/27] bpo-24658: just a test --- Lib/test/test_largefile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 634f8b0698f71a..a42833f2101acd 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,9 +45,9 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - @bigmemtest(size=_2G, memuse=1) - def test_large_reads_writes(self, _size): + def test_large_reads_writes(self): # see issue #24658 + size = size / 10 requires('largefile', 'test requires %s bytes and a long time to run' % size) self.addCleanup(unlink, TESTFN) From 7064bd55e43d5285d13f81edf74e82f99100b1a5 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 01:27:57 +0200 Subject: [PATCH 07/27] bpo-24658: just a test --- Lib/test/test_largefile.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index a42833f2101acd..f091b6447f1a13 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -47,18 +47,22 @@ def tearDownClass(cls): def test_large_reads_writes(self): # see issue #24658 - size = size / 10 requires('largefile', 'test requires %s bytes and a long time to run' % size) - self.addCleanup(unlink, TESTFN) + # self.addCleanup(unlink, TESTFN) + # with self.open(TESTFN, "wb") as f: + # b = b'x' * size + # self.assertEqual(f.write(b), size) + # self.assertEqual(f.tell(), size) with self.open(TESTFN, "wb") as f: - b = b'x' * size - self.assertEqual(f.write(b), size) - self.assertEqual(f.tell(), size) + f.seek(size) + f.write(b'a') + f.flush() + self.assertEqual(f.tell(), size + 1) with self.open(TESTFN, "rb") as f: f.read() - self.assertEqual(f.tell(), size) + self.assertEqual(f.tell(), size + 1) def test_osstat(self): self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) From 84b09f92addcdd37ff1b43b3f5a8c0c7bb343693 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 01:37:30 +0200 Subject: [PATCH 08/27] bpo-24658: just a test --- Lib/test/test_largefile.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index f091b6447f1a13..ad45716b60c4ed 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -49,19 +49,9 @@ def test_large_reads_writes(self): # see issue #24658 requires('largefile', 'test requires %s bytes and a long time to run' % size) - # self.addCleanup(unlink, TESTFN) - # with self.open(TESTFN, "wb") as f: - # b = b'x' * size - # self.assertEqual(f.write(b), size) - # self.assertEqual(f.tell(), size) - with self.open(TESTFN, "wb") as f: - f.seek(size) - f.write(b'a') - f.flush() - self.assertEqual(f.tell(), size + 1) with self.open(TESTFN, "rb") as f: - f.read() + self.assertEqual(len(f.read()), size + 1) self.assertEqual(f.tell(), size + 1) def test_osstat(self): From a8c0aa1a57ebc051563a6be624ee10ebafa9a03a Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 01:45:12 +0200 Subject: [PATCH 09/27] bpo-24658: remove the requires and fix the comment --- Lib/test/test_largefile.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index ad45716b60c4ed..911da8bc6cf496 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -46,10 +46,7 @@ def tearDownClass(cls): 'with mode "wb"') def test_large_reads_writes(self): - # see issue #24658 - requires('largefile', - 'test requires %s bytes and a long time to run' % size) - + # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: self.assertEqual(len(f.read()), size + 1) self.assertEqual(f.tell(), size + 1) From 3d7fe38a31cef252294fef17baec94efd46232fe Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 01:46:34 +0200 Subject: [PATCH 10/27] bpo-24658: rename the test method --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 911da8bc6cf496..24dbfb41cdb0c6 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,7 +45,7 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - def test_large_reads_writes(self): + def test_large_read(self): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: self.assertEqual(len(f.read()), size + 1) From bb3e0826242179f77ed5544c417a37630c2fd567 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 18:42:27 +0200 Subject: [PATCH 11/27] bpo-24658: remove unused imports --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 24dbfb41cdb0c6..867319ee1660d9 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -5,7 +5,7 @@ import stat import sys import unittest -from test.support import TESTFN, requires, unlink, bigmemtest, _2G +from test.support import TESTFN, requires, unlink import io # C implementation of io import _pyio as pyio # Python implementation of io From fc08e60bbe439e73abf20b8b0a2f3ad1cb05f0c7 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Tue, 23 May 2017 21:26:39 +0200 Subject: [PATCH 12/27] bpo-24658: check with bigmemtest --- Lib/test/test_largefile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 867319ee1660d9..f39cceee8a9da2 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,7 +45,8 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - def test_large_read(self): + @bigmemtest(size=_2G, memuse=1) + def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: self.assertEqual(len(f.read()), size + 1) From 17d0d043fcd6d444fcac19e08276cdaeb68eb978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 14:36:19 +0200 Subject: [PATCH 13/27] Add blurb entry --- Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst diff --git a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst new file mode 100644 index 00000000000000..5c04557cb0c2b4 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst @@ -0,0 +1 @@ +Fix read/write on file with a size > 2GB on macOS From 468dbbe6edd0c22cc71164cc2636aa1462e82e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 14:56:06 +0200 Subject: [PATCH 14/27] Add the missing symbols from test.support --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index f39cceee8a9da2..448023ce93ce38 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -5,7 +5,7 @@ import stat import sys import unittest -from test.support import TESTFN, requires, unlink +from test.support import TESTFN, requires, unlink, bigmemtest, _2G import io # C implementation of io import _pyio as pyio # Python implementation of io From 4140591e26a3fb9bcc769fbe29986565b7cc4c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 16:23:46 +0200 Subject: [PATCH 15/27] Update with the review of Victor --- Include/fileutils.h | 2 ++ Lib/test/test_largefile.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 9b0f93e2535a22..1687ac0dd09741 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -90,6 +90,8 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #else #define _PY_READ_MAX PY_SSIZE_T_MAX #define _PY_WRITE_MAX PY_SSIZE_T_MAX +/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer + * to do it ourself to have a portable behaviour */ #endif #ifdef MS_WINDOWS diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 448023ce93ce38..c23036b2a2c451 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -10,7 +10,7 @@ import _pyio as pyio # Python implementation of io # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes) -size = 2500000000 +size = 2_500_000_000 class LargeFileTest: """Test that each file function works as expected for large @@ -45,7 +45,7 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - @bigmemtest(size=_2G, memuse=1) + @bigmemtest(size=size+1, memuse=1) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: From 86fa808f9bfba6abe73760373f369654c654ea2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 16:28:26 +0200 Subject: [PATCH 16/27] Update with the review of Victor --- Include/fileutils.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 1687ac0dd09741..32f139a7e41689 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -83,13 +83,12 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) /* On Windows, the count parameter of read() is an int. - macOS fails when reading or writing more than 2GB, see #24658 -*/ -#define _PY_READ_MAX INT_MAX -#define _PY_WRITE_MAX INT_MAX + * macOS fails when reading or writing more than 2GB, see bpo-24658 */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX #else -#define _PY_READ_MAX PY_SSIZE_T_MAX -#define _PY_WRITE_MAX PY_SSIZE_T_MAX +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX /* write() should truncate count to PY_SSIZE_T_MAX, but it's safer * to do it ourself to have a portable behaviour */ #endif From 83dd75976cb715ca375e50111d7d698cd9309277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 16:31:10 +0200 Subject: [PATCH 17/27] Remove _2G, not used --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index c23036b2a2c451..628a2d90b8720c 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -5,7 +5,7 @@ import stat import sys import unittest -from test.support import TESTFN, requires, unlink, bigmemtest, _2G +from test.support import TESTFN, requires, unlink, bigmemtest import io # C implementation of io import _pyio as pyio # Python implementation of io From 15e176d7ee35a0fa872685fa47490c0c60f11eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 16:39:44 +0200 Subject: [PATCH 18/27] Update with the recommendations of Serhiy --- Lib/test/test_largefile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 628a2d90b8720c..c4788507237621 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,11 +45,11 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - @bigmemtest(size=size+1, memuse=1) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: - self.assertEqual(len(f.read()), size + 1) + self.assertEqual(len(f.read()), size) self.assertEqual(f.tell(), size + 1) def test_osstat(self): From d4ec7ec1b7732e79abff5043dd4cdcef2b5cee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 22:47:12 +0200 Subject: [PATCH 19/27] fix test_large_read --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index c4788507237621..98402270bc355b 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -49,7 +49,7 @@ def tearDownClass(cls): def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: - self.assertEqual(len(f.read()), size) + self.assertEqual(len(f.read()), size + 1) self.assertEqual(f.tell(), size + 1) def test_osstat(self): From a21bfcbefbb94d6bc67a90b1bcf519761153dfab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 22:58:39 +0200 Subject: [PATCH 20/27] PEP 7 for the comments --- Include/fileutils.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index 32f139a7e41689..ba289045fc140f 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -82,15 +82,14 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) -/* On Windows, the count parameter of read() is an int. - * macOS fails when reading or writing more than 2GB, see bpo-24658 */ + /* On Windows, the count parameter of read() is an int. + macOS fails when reading or writing more than 2GB, see bpo-24658 */ # define _PY_READ_MAX INT_MAX # define _PY_WRITE_MAX INT_MAX #else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes */ # define _PY_READ_MAX PY_SSIZE_T_MAX # define _PY_WRITE_MAX PY_SSIZE_T_MAX -/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer - * to do it ourself to have a portable behaviour */ #endif #ifdef MS_WINDOWS From 4382256f8d7cd1622a34a9c9c51c927bfe64c667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 23:11:30 +0200 Subject: [PATCH 21/27] Use memuse=2 --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 98402270bc355b..a4770c3d6f0430 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,7 +45,7 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - @bigmemtest(size=size, memuse=1, dry_run=False) + @bigmemtest(size=size, memuse=2, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: From a6968997a08d80f767920b5e2a698973d8542299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 23:25:17 +0200 Subject: [PATCH 22/27] Add comments --- Include/fileutils.h | 3 ++- Lib/test/test_largefile.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index ba289045fc140f..f53ce6dcf3a2ba 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -87,7 +87,8 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); # define _PY_READ_MAX INT_MAX # define _PY_WRITE_MAX INT_MAX #else - /* write() should truncate the input to PY_SSIZE_T_MAX bytes */ + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ # define _PY_READ_MAX PY_SSIZE_T_MAX # define _PY_WRITE_MAX PY_SSIZE_T_MAX #endif diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index a4770c3d6f0430..6e16b1e3912b9e 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,6 +45,8 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') + # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes + # so memuse=2 is needed @bigmemtest(size=size, memuse=2, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. From b8d1f243a61bfec205f97d7d3a963305201fa79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 17 Oct 2018 23:27:44 +0200 Subject: [PATCH 23/27] Update the NEWS entry --- Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst index 5c04557cb0c2b4..c83a87ba3c1915 100644 --- a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst +++ b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst @@ -1 +1 @@ -Fix read/write on file with a size > 2GB on macOS +On macOS, fix reading from and writing into a file with a size larger than 2GiB. \ No newline at end of file From 3e887f5f6cdc0ae63efcf11d43b47d4e4634cd87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 00:42:42 +0200 Subject: [PATCH 24/27] Update the comments about the issues with read()/write() on macOS and Windows --- Include/fileutils.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index f53ce6dcf3a2ba..c8d2810f32fd32 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -82,8 +82,9 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) - /* On Windows, the count parameter of read() is an int. - macOS fails when reading or writing more than 2GB, see bpo-24658 */ + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fails with EINVAL (bpo-24658). */ # define _PY_READ_MAX INT_MAX # define _PY_WRITE_MAX INT_MAX #else From 530799a6a890e9dfd4633511cf6a2aba892461d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 00:47:09 +0200 Subject: [PATCH 25/27] Fix typo --- Include/fileutils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index c8d2810f32fd32..232d9664cdd013 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -84,7 +84,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). On macOS 10.13, read() and write() with more than INT_MAX bytes - fails with EINVAL (bpo-24658). */ + fail with EINVAL (bpo-24658). */ # define _PY_READ_MAX INT_MAX # define _PY_WRITE_MAX INT_MAX #else From 4c9a61f068f78c47f138e9992b882bc34c215ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 00:47:31 +0200 Subject: [PATCH 26/27] add comma --- Lib/test/test_largefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 6e16b1e3912b9e..8870c721ab0efb 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -45,7 +45,7 @@ def tearDownClass(cls): raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes + # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, # so memuse=2 is needed @bigmemtest(size=size, memuse=2, dry_run=False) def test_large_read(self, _size): From 04ccd69e106f8ce414d230881e740a68c11ec2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Thu, 18 Oct 2018 00:48:12 +0200 Subject: [PATCH 27/27] add space between 2 and GiB --- Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst index c83a87ba3c1915..ff660a125c64f3 100644 --- a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst +++ b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst @@ -1 +1 @@ -On macOS, fix reading from and writing into a file with a size larger than 2GiB. \ No newline at end of file +On macOS, fix reading from and writing into a file with a size larger than 2 GiB. \ No newline at end of file