From b3459cb7b5934d7b5d5de6842a08828828df36eb Mon Sep 17 00:00:00 2001 From: Jeffrey Kintscher Date: Mon, 8 Jul 2019 19:09:07 -0700 Subject: [PATCH 1/2] bpo-37367: raise ValueError for byte strings containing octal values greater than 255 (0o377) --- Lib/test/test_bytes.py | 38 +++++++++++++++++++ Lib/test/test_codecs.py | 2 +- .../2019-07-08-19-08-38.bpo-37367.SVtBhh.rst | 2 + Objects/bytesobject.c | 5 +++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b5eeb2b4fc2901..cd6fd1c6b4588a 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -14,6 +14,7 @@ import tempfile import textwrap import unittest +from ast import literal_eval as _literal_eval import test.support import test.string_tests @@ -962,6 +963,43 @@ def test_translate(self): c = b.translate(None, delete=b'e') self.assertEqual(c, b'hllo') + def test_octal_values(self): + # bpo-37367: verify that octal values greater than 255 + # raise ValueError + + # test 1- and 2- digit octal values (i.e. no leading zeroes) + for i in range(0o00, 0o77): + v = "b'\\{0:o}'".format(i) + self.type2test(_literal_eval(v)) + self.assertEqual(ord(_literal_eval(v)), i) + + # test 3-digit octal values (including leading zeroes) + for i in range(0o00, 0o400): + v = "b'\\{0:03o}'".format(i) + self.type2test(_literal_eval(v)) + self.assertEqual(ord(_literal_eval(v)), i) + + raised = 0 + for i in range(0o400, 0o1000): + try: + self.type2test(_literal_eval("b'\\{0:o}'".format(i))) + except SyntaxError as e: + # ast.literal_eval() raises SyntaxError and + # mentions the underlying ValueError in + # the returned string. + self.assertEqual(str(e).find("(value error) octal value " + "must be in range(0, 256)", 0), + 0) + raised = raised + 1 + self.assertEqual(raised, 256) + + # test 4-digit octal value (4th digit should be treated as literal) + self.assertEqual(_literal_eval("b'\\1234'"), b'S4') + self.assertRaises(SyntaxError, _literal_eval, "b'\\4321'") + + # specific case mentioned in bpo-37367 + self.assertRaises(SyntaxError, eval, "ord(b'\\407')") + class BytesTest(BaseBytesTest, unittest.TestCase): type2test = bytes diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index b187ca650dc693..c09f2e2eb322ef 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1174,7 +1174,7 @@ def test_escape(self): check(br"[\418]", b"[!8]") check(br"[\101]", b"[A]") check(br"[\1010]", b"[A0]") - check(br"[\501]", b"[A]") + self.assertRaises(ValueError, decode, br"[\501]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") for i in range(97, 123): diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst b/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst new file mode 100644 index 00000000000000..16a000cff54b41 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst @@ -0,0 +1,2 @@ +Byte strings containing octal values greater than 255 will now raise +ValueError. Patch by (Jeffrey Kintscher). diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c4edcca4f76127..c365e97015b3b4 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1185,6 +1185,11 @@ PyObject *_PyBytes_DecodeEscape(const char *s, if (s < end && '0' <= *s && *s <= '7') c = (c<<3) + *s++ - '0'; } + if (c > 255) { + PyErr_Format(PyExc_ValueError, + "octal value must be in range(0, 256)"); + goto failed; + } *p++ = c; break; case 'x': From b521f7191e2296784431925ebbf76c952ba80c2c Mon Sep 17 00:00:00 2001 From: Jeffrey Kintscher Date: Mon, 15 Jul 2019 17:42:09 -0700 Subject: [PATCH 2/2] bpo-37367: changes per reviewer request: 1. use "octal escape sequences" instead of "octal values" in comments 2. remove underscore from _literal_eval Also, use "0o1000-0o400" instead of "256" to make it clear to the reader where the value came from. --- Lib/test/test_bytes.py | 28 ++++++++++--------- .../2019-07-08-19-08-38.bpo-37367.SVtBhh.rst | 4 +-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index cd6fd1c6b4588a..3144a520dd250a 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -14,7 +14,7 @@ import tempfile import textwrap import unittest -from ast import literal_eval as _literal_eval +import ast import test.support import test.string_tests @@ -964,25 +964,27 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_octal_values(self): - # bpo-37367: verify that octal values greater than 255 - # raise ValueError + # bpo-37367: verify that octal escape sequences + # with values greater than 255 raise ValueError - # test 1- and 2- digit octal values (i.e. no leading zeroes) + # test 1- and 2- digit octal escape sequences + # (i.e. no leading zeroes) for i in range(0o00, 0o77): v = "b'\\{0:o}'".format(i) - self.type2test(_literal_eval(v)) - self.assertEqual(ord(_literal_eval(v)), i) + self.type2test(ast.literal_eval(v)) + self.assertEqual(ord(ast.literal_eval(v)), i) - # test 3-digit octal values (including leading zeroes) + # test 3-digit octal escape sequences + # (including leading zeroes) for i in range(0o00, 0o400): v = "b'\\{0:03o}'".format(i) - self.type2test(_literal_eval(v)) - self.assertEqual(ord(_literal_eval(v)), i) + self.type2test(ast.literal_eval(v)) + self.assertEqual(ord(ast.literal_eval(v)), i) raised = 0 for i in range(0o400, 0o1000): try: - self.type2test(_literal_eval("b'\\{0:o}'".format(i))) + self.type2test(ast.literal_eval("b'\\{0:o}'".format(i))) except SyntaxError as e: # ast.literal_eval() raises SyntaxError and # mentions the underlying ValueError in @@ -991,11 +993,11 @@ def test_octal_values(self): "must be in range(0, 256)", 0), 0) raised = raised + 1 - self.assertEqual(raised, 256) + self.assertEqual(raised, 0o1000-0o400) # test 4-digit octal value (4th digit should be treated as literal) - self.assertEqual(_literal_eval("b'\\1234'"), b'S4') - self.assertRaises(SyntaxError, _literal_eval, "b'\\4321'") + self.assertEqual(ast.literal_eval("b'\\1234'"), b'S4') + self.assertRaises(SyntaxError, ast.literal_eval, "b'\\4321'") # specific case mentioned in bpo-37367 self.assertRaises(SyntaxError, eval, "ord(b'\\407')") diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst b/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst index 16a000cff54b41..14fdb1c390dae6 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-07-08-19-08-38.bpo-37367.SVtBhh.rst @@ -1,2 +1,2 @@ -Byte strings containing octal values greater than 255 will now raise -ValueError. Patch by (Jeffrey Kintscher). +Byte strings containing octal escape sequences with values greater than 255 +will now raise ValueError. Patch by Jeffrey Kintscher.