Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 687ff0e

Browse files
serhiy-storchakaserhiy-storchaka
committed
Issue #11489: JSON decoder now accepts lone surrogates.
2 parents 1df8867 + c93329b commit 687ff0e

4 files changed

Lines changed: 73 additions & 41 deletions

File tree

‎Lib/json/decoder.py‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ def errmsg(msg, doc, pos, end=None):
5858
'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
5959
}
6060

61+
def _decode_uXXXX(s, pos):
62+
esc = s[pos + 1:pos + 5]
63+
if len(esc) == 4 and esc[1] not in 'xX':
64+
try:
65+
return int(esc, 16)
66+
except ValueError:
67+
pass
68+
msg = "Invalid \\uXXXX escape"
69+
raise ValueError(errmsg(msg, s, pos))
70+
6171
def py_scanstring(s, end, strict=True,
6272
_b=BACKSLASH, _m=STRINGCHUNK.match):
6373
"""Scan the string s for a JSON string. End is the index of the
@@ -107,25 +117,14 @@ def py_scanstring(s, end, strict=True,
107117
raise ValueError(errmsg(msg, s, end))
108118
end += 1
109119
else:
110-
esc = s[end + 1:end + 5]
111-
next_end = end + 5
112-
if len(esc) != 4:
113-
msg = "Invalid \\uXXXX escape"
114-
raise ValueError(errmsg(msg, s, end))
115-
uni = int(esc, 16)
116-
if 0xd800 <= uni <= 0xdbff:
117-
msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
118-
if not s[end + 5:end + 7] == '\\u':
119-
raise ValueError(errmsg(msg, s, end))
120-
esc2 = s[end + 7:end + 11]
121-
if len(esc2) != 4:
122-
raise ValueError(errmsg(msg, s, end))
123-
uni2 = int(esc2, 16)
124-
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
125-
next_end += 6
120+
uni = _decode_uXXXX(s, end)
121+
end += 5
122+
if 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u':
123+
uni2 = _decode_uXXXX(s, end + 1)
124+
if 0xdc00 <= uni2 <= 0xdfff:
125+
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
126+
end += 6
126127
char = chr(uni)
127-
128-
end = next_end
129128
_append(char)
130129
return ''.join(chunks), end
131130

‎Lib/test/test_json/test_scanstring.py‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
class TestScanstring:
66
def test_scanstring(self):
77
scanstring = self.json.decoder.scanstring
8-
self.assertEqual(
9-
scanstring('"z\\ud834\\udd20x"', 1, True),
10-
('z\U0001d120x', 16))
11-
128
self.assertEqual(
139
scanstring('"z\U0001d120x"', 1, True),
1410
('z\U0001d120x', 5))
@@ -89,6 +85,53 @@ def test_scanstring(self):
8985
scanstring('["Bad value", truth]', 2, True),
9086
('Bad value', 12))
9187

88+
def test_surrogates(self):
89+
scanstring = self.json.decoder.scanstring
90+
def assertScan(given, expect):
91+
self.assertEqual(scanstring(given, 1, True),
92+
(expect, len(given)))
93+
94+
assertScan('"z\\ud834\\u0079x"', 'z\ud834yx')
95+
assertScan('"z\\ud834\\udd20x"', 'z\U0001d120x')
96+
assertScan('"z\\ud834\\ud834\\udd20x"', 'z\ud834\U0001d120x')
97+
assertScan('"z\\ud834x"', 'z\ud834x')
98+
assertScan('"z\\ud834\udd20x12345"', 'z\ud834\udd20x12345')
99+
assertScan('"z\\udd20x"', 'z\udd20x')
100+
assertScan('"z\ud834\udd20x"', 'z\ud834\udd20x')
101+
assertScan('"z\ud834\\udd20x"', 'z\ud834\udd20x')
102+
assertScan('"z\ud834x"', 'z\ud834x')
103+
104+
def test_bad_escapes(self):
105+
scanstring = self.json.decoder.scanstring
106+
bad_escapes = [
107+
'"\\"',
108+
'"\\x"',
109+
'"\\u"',
110+
'"\\u0"',
111+
'"\\u01"',
112+
'"\\u012"',
113+
'"\\uz012"',
114+
'"\\u0z12"',
115+
'"\\u01z2"',
116+
'"\\u012z"',
117+
'"\\u0x12"',
118+
'"\\u0X12"',
119+
'"\\ud834\\"',
120+
'"\\ud834\\u"',
121+
'"\\ud834\\ud"',
122+
'"\\ud834\\udd"',
123+
'"\\ud834\\udd2"',
124+
'"\\ud834\\uzdd2"',
125+
'"\\ud834\\udzd2"',
126+
'"\\ud834\\uddz2"',
127+
'"\\ud834\\udd2z"',
128+
'"\\ud834\\u0x20"',
129+
'"\\ud834\\u0X20"',
130+
]
131+
for s in bad_escapes:
132+
with self.assertRaises(ValueError, msg=s):
133+
scanstring(s, 1, True)
134+
92135
def test_overflow(self):
93136
with self.assertRaises(OverflowError):
94137
self.json.decoder.scanstring(b"xxx", sys.maxsize+1)

‎Misc/NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ Core and Builtins
1616
Library
1717
-------
1818

19+
- Issue #11489: JSON decoder now accepts lone surrogates.
20+
1921
- Issue #19545: Avoid chained exceptions while passing stray % to
2022
time.strptime(). Initial patch by Claudiu Popa.
2123

‎Modules/_json.c‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -409,17 +409,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
409409
}
410410
}
411411
/* Surrogate pair */
412-
if (Py_UNICODE_IS_HIGH_SURROGATE(c)) {
412+
if (Py_UNICODE_IS_HIGH_SURROGATE(c) && end + 6 < len &&
413+
PyUnicode_READ(kind, buf, next++) == '\\' &&
414+
PyUnicode_READ(kind, buf, next++) == 'u') {
413415
Py_UCS4 c2 = 0;
414-
if (end + 6 >= len) {
415-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
416-
goto bail;
417-
}
418-
if (PyUnicode_READ(kind, buf, next++) != '\\' ||
419-
PyUnicode_READ(kind, buf, next++) != 'u') {
420-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
421-
goto bail;
422-
}
423416
end += 6;
424417
/* Decode 4 hex digits */
425418
for (; next < end; next++) {
@@ -440,15 +433,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next
440433
goto bail;
441434
}
442435
}
443-
if (!Py_UNICODE_IS_LOW_SURROGATE(c2)) {
444-
raise_errmsg("Unpaired high surrogate", pystr, end - 5);
445-
goto bail;
446-
}
447-
c = Py_UNICODE_JOIN_SURROGATES(c, c2);
448-
}
449-
else if (Py_UNICODE_IS_LOW_SURROGATE(c)) {
450-
raise_errmsg("Unpaired low surrogate", pystr, end - 5);
451-
goto bail;
436+
if (Py_UNICODE_IS_LOW_SURROGATE(c2))
437+
c = Py_UNICODE_JOIN_SURROGATES(c, c2);
438+
else
439+
end -= 6;
452440
}
453441
}
454442
APPEND_OLD_CHUNK

0 commit comments

Comments
 (0)