Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f86a5e8

Browse files
vstinnervstinner
committed
Close #11022: TextIOWrapper doesn't call locale.setlocale() anymore
open() and io.TextIOWrapper are now calling locale.getpreferredencoding(False) instead of locale.getpreferredencoding() in text mode if the encoding is not specified. Don't change temporary the locale encoding using locale.setlocale(), use the current locale encoding instead of the user preferred encoding. Explain also in open() documentation that locale.getpreferredencoding(False) is called if the encoding is not specified.
1 parent 91c5a34 commit f86a5e8

8 files changed

Lines changed: 82 additions & 27 deletions

File tree

‎Doc/library/functions.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -800,9 +800,10 @@ are always available. They are listed here in alphabetical order.
800800
already exists), ``'x'`` for exclusive creation and ``'a'`` for appending
801801
(which on *some* Unix systems, means that *all* writes append to the end of
802802
the file regardless of the current seek position). In text mode, if
803-
*encoding* is not specified the encoding used is platform dependent. (For
804-
reading and writing raw bytes use binary mode and leave *encoding*
805-
unspecified.) The available modes are:
803+
*encoding* is not specified the encoding used is platform dependent:
804+
``locale.getpreferredencoding(False)`` is called to get the current locale
805+
encoding. (For reading and writing raw bytes use binary mode and leave
806+
*encoding* unspecified.) The available modes are:
806807

807808
========= ===============================================================
808809
Character Meaning

‎Doc/library/io.rst‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ Text I/O
752752
It inherits :class:`TextIOBase`.
753753

754754
*encoding* gives the name of the encoding that the stream will be decoded or
755-
encoded with. It defaults to :func:`locale.getpreferredencoding`.
755+
encoded with. It defaults to ``locale.getpreferredencoding(False)``.
756756

757757
*errors* is an optional string that specifies how encoding and decoding
758758
errors are to be handled. Pass ``'strict'`` to raise a :exc:`ValueError`
@@ -784,6 +784,12 @@ Text I/O
784784
.. versionchanged:: 3.3
785785
The *write_through* argument has been added.
786786

787+
.. versionchanged:: 3.3
788+
The default *encoding* is now ``locale.getpreferredencoding(False)``
789+
instead of ``locale.getpreferredencoding()``. Don't change temporary the
790+
locale encoding using :func:`locale.setlocale`, use the current locale
791+
encoding instead of the user preferred encoding.
792+
787793
:class:`TextIOWrapper` provides one attribute in addition to those of
788794
:class:`TextIOBase` and its parents:
789795

‎Lib/_pyio.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1448,7 +1448,7 @@ class TextIOWrapper(TextIOBase):
14481448
r"""Character and line based layer over a BufferedIOBase object, buffer.
14491449
14501450
encoding gives the name of the encoding that the stream will be
1451-
decoded or encoded with. It defaults to locale.getpreferredencoding.
1451+
decoded or encoded with. It defaults to locale.getpreferredencoding(False).
14521452
14531453
errors determines the strictness of encoding and decoding (see the
14541454
codecs.register) and defaults to "strict".
@@ -1487,7 +1487,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None,
14871487
# Importing locale may fail if Python is being built
14881488
encoding = "ascii"
14891489
else:
1490-
encoding = locale.getpreferredencoding()
1490+
encoding = locale.getpreferredencoding(False)
14911491

14921492
if not isinstance(encoding, str):
14931493
raise ValueError("invalid encoding: %r" % encoding)

‎Lib/test/test_builtin.py‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
# Python test set -- built-in functions
22

3-
import platform
4-
import unittest
5-
import sys
6-
import warnings
3+
import ast
4+
import builtins
75
import collections
86
import io
7+
import locale
98
import os
10-
import ast
11-
import types
12-
import builtins
9+
import pickle
10+
import platform
1311
import random
12+
import sys
1413
import traceback
15-
from test.support import TESTFN, unlink, run_unittest, check_warnings
14+
import types
15+
import unittest
16+
import warnings
1617
from operator import neg
17-
import pickle
18+
from test.support import TESTFN, unlink, run_unittest, check_warnings
1819
try:
1920
import pty, signal
2021
except ImportError:
@@ -961,6 +962,27 @@ def test_open(self):
961962
fp.close()
962963
unlink(TESTFN)
963964

965+
def test_open_default_encoding(self):
966+
old_environ = dict(os.environ)
967+
try:
968+
# try to get a user preferred encoding different than the current
969+
# locale encoding to check that open() uses the current locale
970+
# encoding and not the user preferred encoding
971+
for key in ('LC_ALL', 'LANG', 'LC_CTYPE'):
972+
if key in os.environ:
973+
del os.environ[key]
974+
975+
self.write_testfile()
976+
current_locale_encoding = locale.getpreferredencoding(False)
977+
fp = open(TESTFN, 'w')
978+
try:
979+
self.assertEqual(fp.encoding, current_locale_encoding)
980+
finally:
981+
fp.close()
982+
finally:
983+
os.environ.clear()
984+
os.environ.update(old_environ)
985+
964986
def test_ord(self):
965987
self.assertEqual(ord(' '), 32)
966988
self.assertEqual(ord('A'), 65)

‎Lib/test/test_io.py‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,21 @@
1919
# test both implementations. This file has lots of examples.
2020
################################################################################
2121

22+
import abc
23+
import array
24+
import errno
25+
import locale
2226
import os
27+
import pickle
28+
import random
29+
import signal
2330
import sys
2431
import time
25-
import array
26-
import random
2732
import unittest
28-
import weakref
29-
import abc
30-
import signal
31-
import errno
3233
import warnings
33-
import pickle
34-
from itertools import cycle, count
34+
import weakref
3535
from collections import deque
36+
from itertools import cycle, count
3637
from test import support
3738

3839
import codecs
@@ -1881,6 +1882,24 @@ def test_line_buffering(self):
18811882
t.write("A\rB")
18821883
self.assertEqual(r.getvalue(), b"XY\nZA\rB")
18831884

1885+
def test_default_encoding(self):
1886+
old_environ = dict(os.environ)
1887+
try:
1888+
# try to get a user preferred encoding different than the current
1889+
# locale encoding to check that TextIOWrapper() uses the current
1890+
# locale encoding and not the user preferred encoding
1891+
for key in ('LC_ALL', 'LANG', 'LC_CTYPE'):
1892+
if key in os.environ:
1893+
del os.environ[key]
1894+
1895+
current_locale_encoding = locale.getpreferredencoding(False)
1896+
b = self.BytesIO()
1897+
t = self.TextIOWrapper(b)
1898+
self.assertEqual(t.encoding, current_locale_encoding)
1899+
finally:
1900+
os.environ.clear()
1901+
os.environ.update(old_environ)
1902+
18841903
def test_encoding(self):
18851904
# Check the encoding attribute is always set, and valid
18861905
b = self.BytesIO()

‎Misc/NEWS‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ What's New in Python 3.3.0 Beta 1?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #11022: open() and io.TextIOWrapper are now calling
14+
locale.getpreferredencoding(False) instead of locale.getpreferredencoding()
15+
in text mode if the encoding is not specified. Don't change temporary the
16+
locale encoding using locale.setlocale(), use the current locale encoding
17+
instead of the user preferred encoding.
18+
1319
- Issue #14673: Add Eric Snow's sys.implementation implementation.
1420

1521
Library

‎Modules/_io/_iomodule.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,9 @@ PyDoc_STRVAR(open_doc,
112112
"'a' for appending (which on some Unix systems, means that all writes\n"
113113
"append to the end of the file regardless of the current seek position).\n"
114114
"In text mode, if encoding is not specified the encoding used is platform\n"
115-
"dependent. (For reading and writing raw bytes use binary mode and leave\n"
116-
"encoding unspecified.) The available modes are:\n"
115+
"dependent: locale.getpreferredencoding(False) is called to get the\n"
116+
"current locale encoding. (For reading and writing raw bytes use binary\n"
117+
"mode and leave encoding unspecified.) The available modes are:\n"
117118
"\n"
118119
"========= ===============================================================\n"
119120
"Character Meaning\n"

‎Modules/_io/textio.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -630,7 +630,7 @@ PyDoc_STRVAR(textiowrapper_doc,
630630
"Character and line based layer over a BufferedIOBase object, buffer.\n"
631631
"\n"
632632
"encoding gives the name of the encoding that the stream will be\n"
633-
"decoded or encoded with. It defaults to locale.getpreferredencoding.\n"
633+
"decoded or encoded with. It defaults to locale.getpreferredencoding(False).\n"
634634
"\n"
635635
"errors determines the strictness of encoding and decoding (see the\n"
636636
"codecs.register) and defaults to \"strict\".\n"
@@ -898,7 +898,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
898898
else {
899899
use_locale:
900900
self->encoding = _PyObject_CallMethodId(
901-
state->locale_module, &PyId_getpreferredencoding, NULL);
901+
state->locale_module, &PyId_getpreferredencoding, "O", Py_False);
902902
if (self->encoding == NULL) {
903903
catch_ImportError:
904904
/*

0 commit comments

Comments
 (0)