From 07cc7dcfe857daa366b95d56f525a24b09eecb12 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 10 Feb 2024 14:50:29 +0000 Subject: [PATCH 01/11] gh-98306: Support JSON encoding of NaNs and infinities as null --- Doc/library/json.rst | 30 ++++++++++++----- Doc/whatsnew/3.13.rst | 8 +++++ Lib/json/__init__.py | 2 +- Lib/json/encoder.py | 5 ++- Lib/test/test_json/test_float.py | 32 +++++++++++++++++++ ...4-02-10-14-48-03.gh-issue-98306.dSlaEP.rst | 2 ++ Modules/_json.c | 24 +++++++++++--- 7 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3..aa2b0048fb9e45 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -172,9 +172,14 @@ Basic Usage If *allow_nan* is false (default: ``True``), then it will be a :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, - ``inf``, ``-inf``) in strict compliance of the JSON specification. - If *allow_nan* is true, their JavaScript equivalents (``NaN``, - ``Infinity``, ``-Infinity``) will be used. + ``inf``, ``-inf``) in strict compliance with the JSON specification. If + *allow_nan* is the string ``"null"``, NaNs and infinities will be converted + to a JSON ``null``, matching the behavior of JavaScript's + ``JSON.stringify``. If *allow_nan* is true but not equal to ``"null"`` then + NaNs and infinities are converted to non-quote-delimited strings ``NaN``, + ``Infinity`` and ``-Infinity`` in the JSON output. Note that this represents + an extension of the JSON specification, and is not compliant with standard + JSON. If *indent* is a non-negative integer or string, then JSON array elements and object members will be pretty-printed with that indent level. An indent level @@ -215,6 +220,9 @@ Basic Usage so trying to serialize multiple objects with repeated calls to :func:`dump` using the same *fp* will result in an invalid JSON file. + .. versionchanged:: 3.13 + Added support for ``allow_nan='null'``. + .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ indent=None, separators=None, default=None, \ @@ -450,11 +458,15 @@ Encoders and Decoders prevent an infinite recursion (which would cause a :exc:`RecursionError`). Otherwise, no such check takes place. - If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and - ``-Infinity`` will be encoded as such. This behavior is not JSON - specification compliant, but is consistent with most JavaScript based - encoders and decoders. Otherwise, it will be a :exc:`ValueError` to encode - such floats. + If *allow_nan* is the string ``"null"``, then NaNs and infinities are + encoded as JSON ``null`` values. This matches the behaviour of JavaScript's + ``JSON.stringify``. If *allow_nan* is true but not equal to ``"null"``, then + ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as corresponding + non-quote-delimited strings in the JSON output. This is the default + behaviour. This behavior represents an extension of the JSON specification, + but is consistent with some JavaScript based encoders and decoders (as well + as Python's own decoder). If *allow_nan* is false, it will be a + :exc:`ValueError` to encode such floats. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that @@ -486,6 +498,8 @@ Encoders and Decoders .. versionchanged:: 3.6 All parameters are now :ref:`keyword-only `. + .. versionchanged:: 3.13 + Added support for ``allow_nan='null'``. .. method:: default(o) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index de79bd979aff80..bc27914e17c0d5 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -279,6 +279,14 @@ ipaddress * Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +json +---- + +* Add support for ``allow_nan='null'`` when encoding to JSON. This converts + floating-point infinities and NaNs to a JSON ``null``, for compatibility + with ECMAScript's ``JSON.stringify``. + (Contributed by Mark Dickinson in :gh:`XXXXXX`.) + marshal ------- diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index ed2c74771ea87d..c3409ee7bcf6b2 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -225,7 +225,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, """ # cached encoder if (not skipkeys and ensure_ascii and - check_circular and allow_nan and + check_circular and allow_nan is True and cls is None and indent is None and separators is None and default is None and not sort_keys and not kw): return _default_encoder.encode(obj) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a8..551f956f8b287d 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -236,7 +236,10 @@ def floatstr(o, allow_nan=self.allow_nan, else: return _repr(o) - if not allow_nan: + if allow_nan == 'null': + return 'null' + + elif not allow_nan: raise ValueError( "Out of range float values are not JSON compliant: " + repr(o)) diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index 61540a3a02c2c6..254fb2aac5f01c 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -2,6 +2,12 @@ from test.test_json import PyTest, CTest +class NotUsableAsABoolean: + def __bool__(self): + raise TypeError("I refuse to be interpreted as a boolean") + + + class TestFloat: def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]: @@ -29,6 +35,32 @@ def test_allow_nan(self): msg = f'Out of range float values are not JSON compliant: {val}' self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) + def test_allow_nan_null(self): + # when allow_nan is "null", infinities and NaNs are converted to "null" + for val in [float('inf'), float('-inf'), float('nan')]: + with self.subTest(val=val): + out = self.dumps([val], allow_nan="null") + res = self.loads(out) + self.assertEqual(res, [None]) + + # and finite values are treated as normal + for val in [1.25, -23, -0.0, 0.0]: + with self.subTest(val=val): + out = self.dumps([val], allow_nan="null") + res = self.loads(out) + self.assertEqual(res, [val]) + + # testing a mixture + vals = [-1.3, 1e100, -math.inf, 1234, -0.0, math.nan] + out = self.dumps(vals, allow_nan="null") + res = self.loads(out) + self.assertEqual(res, [-1.3, 1e100, None, 1234, -0.0, None]) + + def test_allow_nan_non_boolean(self): + # check that exception gets propagated as expected + with self.assertRaises(TypeError): + self.dumps(math.inf, allow_nan=NotUsableAsABoolean()) + class TestPyFloat(TestFloat, PyTest): pass class TestCFloat(TestFloat, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst b/Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst new file mode 100644 index 00000000000000..d752b1fe774cba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst @@ -0,0 +1,2 @@ +Add support for ``allow_nan='null'`` when encoding an object to a JSON +string. This converts floating-point infinities and NaNs to a JSON ``null``. diff --git a/Modules/_json.c b/Modules/_json.c index c55299899e77fe..a6e87031fcca05 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1209,13 +1209,13 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyEncoderObject *s; PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator; + PyObject *item_separator, *allow_nan_obj; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppO:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan)) + &sort_keys, &skipkeys, &allow_nan_obj)) return NULL; if (markers != Py_None && !PyDict_Check(markers)) { @@ -1225,6 +1225,19 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + // allow_nan = + // 0 to disallow nans and infinities + // 1 to convert nans and infinities into corresponding JSON strings + // 2 to convert nans and infinities to a JSON null + if (PyUnicode_Check(allow_nan_obj) && _PyUnicode_Equal(allow_nan_obj, &_Py_ID(null))) { + allow_nan = 2; + } else { + allow_nan = PyObject_IsTrue(allow_nan_obj); + if (allow_nan < 0) { + return NULL; + } + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; @@ -1314,7 +1327,10 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) ); return NULL; } - if (i > 0) { + else if (s->allow_nan == 2) { + return PyUnicode_FromString("null"); + } + else if (i > 0) { return PyUnicode_FromString("Infinity"); } else if (i < 0) { From 42e2543eb3f4a5a507ef864569fc746438311fe3 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 10 Feb 2024 15:04:20 +0000 Subject: [PATCH 02/11] Add PR number --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bc27914e17c0d5..0a297eaf782f1a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -285,7 +285,7 @@ json * Add support for ``allow_nan='null'`` when encoding to JSON. This converts floating-point infinities and NaNs to a JSON ``null``, for compatibility with ECMAScript's ``JSON.stringify``. - (Contributed by Mark Dickinson in :gh:`XXXXXX`.) + (Contributed by Mark Dickinson in :gh:`115246`.) marshal ------- From a72491cddfe249e09df245886da261ebe48971c4 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 10 Feb 2024 15:05:05 +0000 Subject: [PATCH 03/11] Spell behavior consistently --- Doc/library/json.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index aa2b0048fb9e45..dbcce86f1272fe 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -459,11 +459,11 @@ Encoders and Decoders Otherwise, no such check takes place. If *allow_nan* is the string ``"null"``, then NaNs and infinities are - encoded as JSON ``null`` values. This matches the behaviour of JavaScript's + encoded as JSON ``null`` values. This matches the behavior of JavaScript's ``JSON.stringify``. If *allow_nan* is true but not equal to ``"null"``, then ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as corresponding non-quote-delimited strings in the JSON output. This is the default - behaviour. This behavior represents an extension of the JSON specification, + behavior. This behavior represents an extension of the JSON specification, but is consistent with some JavaScript based encoders and decoders (as well as Python's own decoder). If *allow_nan* is false, it will be a :exc:`ValueError` to encode such floats. From b851cafd47d8374349029bdd1af8e302e925c82e Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 10 Feb 2024 15:07:27 +0000 Subject: [PATCH 04/11] Fix inconsistent quote style --- Doc/library/json.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index dbcce86f1272fe..8150c676003f3d 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -173,9 +173,9 @@ Basic Usage If *allow_nan* is false (default: ``True``), then it will be a :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, ``inf``, ``-inf``) in strict compliance with the JSON specification. If - *allow_nan* is the string ``"null"``, NaNs and infinities will be converted + *allow_nan* is the string ``'null'``, NaNs and infinities will be converted to a JSON ``null``, matching the behavior of JavaScript's - ``JSON.stringify``. If *allow_nan* is true but not equal to ``"null"`` then + ``JSON.stringify``. If *allow_nan* is true but not equal to ``'null'`` then NaNs and infinities are converted to non-quote-delimited strings ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this represents an extension of the JSON specification, and is not compliant with standard @@ -458,9 +458,9 @@ Encoders and Decoders prevent an infinite recursion (which would cause a :exc:`RecursionError`). Otherwise, no such check takes place. - If *allow_nan* is the string ``"null"``, then NaNs and infinities are + If *allow_nan* is the string ``'null'``, then NaNs and infinities are encoded as JSON ``null`` values. This matches the behavior of JavaScript's - ``JSON.stringify``. If *allow_nan* is true but not equal to ``"null"``, then + ``JSON.stringify``. If *allow_nan* is true but not equal to ``'null'``, then ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as corresponding non-quote-delimited strings in the JSON output. This is the default behavior. This behavior represents an extension of the JSON specification, From 35e8d330a9fed3cf9d7e998297fb7d9b114f693c Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 10 Feb 2024 15:09:52 +0000 Subject: [PATCH 05/11] Remove extra blank line --- Lib/test/test_json/test_float.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index 254fb2aac5f01c..dec5526db29017 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -7,7 +7,6 @@ def __bool__(self): raise TypeError("I refuse to be interpreted as a boolean") - class TestFloat: def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]: From 2f7f0643cbeb40e5e9c0c9b21eefcc8753a493ba Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 09:25:44 +0100 Subject: [PATCH 06/11] Add a DeprecationWarning for uses of strings other than 'null' --- Doc/library/json.rst | 13 +++++++++---- Lib/json/encoder.py | 10 ++++++++++ Lib/test/test_json/test_float.py | 4 ++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8150c676003f3d..9ae92b6bb8592a 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -214,14 +214,17 @@ Basic Usage .. versionchanged:: 3.6 All optional parameters are now :ref:`keyword-only `. + .. versionchanged:: 3.14 + Added support for ``allow_nan='null'``. Passing any string value + other than ``'null'`` for ``allow_nan`` now raises a + :warning:`DeprecationWarning`. + .. note:: Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, so trying to serialize multiple objects with repeated calls to :func:`dump` using the same *fp* will result in an invalid JSON file. - .. versionchanged:: 3.13 - Added support for ``allow_nan='null'``. .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ @@ -498,8 +501,10 @@ Encoders and Decoders .. versionchanged:: 3.6 All parameters are now :ref:`keyword-only `. - .. versionchanged:: 3.13 - Added support for ``allow_nan='null'``. + .. versionchanged:: 3.14 + Added support for ``allow_nan='null'``. Passing any string value + other than ``'null'`` for ``allow_nan`` now raises a + :warning:`DeprecationWarning`. .. method:: default(o) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 551f956f8b287d..5a2e12aab4f816 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -1,6 +1,7 @@ """Implementation of JSONEncoder """ import re +import warnings try: from _json import encode_basestring_ascii as c_encode_basestring_ascii @@ -148,6 +149,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, self.skipkeys = skipkeys self.ensure_ascii = ensure_ascii self.check_circular = check_circular + + if isinstance(allow_nan, str) and allow_nan != "null": + warnings.warn( + "in the future, allow_nan will no longer accept strings " + "other than 'null'. Use a boolean instead.", + DeprecationWarning, + stacklevel=3, + ) + self.allow_nan = allow_nan self.sort_keys = sort_keys self.indent = indent diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index dec5526db29017..34642a8371c11b 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -55,6 +55,10 @@ def test_allow_nan_null(self): res = self.loads(out) self.assertEqual(res, [-1.3, 1e100, None, 1234, -0.0, None]) + def test_allow_nan_string_deprecation(self): + with self.assertWarns(DeprecationWarning): + self.dumps(2.3, allow_nan="true") + def test_allow_nan_non_boolean(self): # check that exception gets propagated as expected with self.assertRaises(TypeError): From 72673dce9d6320519984134b69a4ac83467c89a2 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 09:27:32 +0100 Subject: [PATCH 07/11] Refresh news entry --- ...6.dSlaEP.rst => 2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/Library/{2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst => 2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst} (100%) diff --git a/Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst b/Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst similarity index 100% rename from Misc/NEWS.d/next/Library/2024-02-10-14-48-03.gh-issue-98306.dSlaEP.rst rename to Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst From 79ce75e59e4251657ebfb1dce5bd2e94c595e1e3 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 09:56:07 +0100 Subject: [PATCH 08/11] Change 'null' to 'as_null' --- Doc/library/json.rst | 38 +++++++++---------- Doc/whatsnew/3.13.rst | 8 ---- Doc/whatsnew/3.14.rst | 9 ++++- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 ++ Lib/json/encoder.py | 6 +-- Lib/test/test_json/test_float.py | 10 ++--- ...4-05-12-09-27-16.gh-issue-98306.dSlaEP.rst | 2 +- Modules/_json.c | 3 +- 11 files changed, 44 insertions(+), 38 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index f856c34e1fedca..fb5c891579a7c1 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -173,13 +173,13 @@ Basic Usage If *allow_nan* is false (default: ``True``), then it will be a :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, ``inf``, ``-inf``) in strict compliance with the JSON specification. If - *allow_nan* is the string ``'null'``, NaNs and infinities will be converted - to a JSON ``null``, matching the behavior of JavaScript's - ``JSON.stringify``. If *allow_nan* is true but not equal to ``'null'`` then - NaNs and infinities are converted to non-quote-delimited strings ``NaN``, - ``Infinity`` and ``-Infinity`` in the JSON output. Note that this represents - an extension of the JSON specification, and is not compliant with standard - JSON. + *allow_nan* is the string ``'as_null'``, NaNs and infinities will be + converted to a JSON ``null``, matching the behavior of JavaScript's + ``JSON.stringify``. If *allow_nan* is true but not equal to ``'as_null'`` + then NaNs and infinities are converted to non-quote-delimited strings + ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this + represents an extension of the JSON specification, and is not compliant with + standard JSON. If *indent* is a non-negative integer or string, then JSON array elements and object members will be pretty-printed with that indent level. An indent level @@ -215,8 +215,8 @@ Basic Usage All optional parameters are now :ref:`keyword-only `. .. versionchanged:: 3.14 - Added support for ``allow_nan='null'``. Passing any string value - other than ``'null'`` for ``allow_nan`` now raises a + Added support for ``allow_nan='as_null'``. Passing any string value + other than ``'as_null'`` for ``allow_nan`` now raises a :warning:`DeprecationWarning`. .. note:: @@ -461,15 +461,15 @@ Encoders and Decoders prevent an infinite recursion (which would cause a :exc:`RecursionError`). Otherwise, no such check takes place. - If *allow_nan* is the string ``'null'``, then NaNs and infinities are + If *allow_nan* is the string ``'as_null'``, then NaNs and infinities are encoded as JSON ``null`` values. This matches the behavior of JavaScript's - ``JSON.stringify``. If *allow_nan* is true but not equal to ``'null'``, then - ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as corresponding - non-quote-delimited strings in the JSON output. This is the default - behavior. This behavior represents an extension of the JSON specification, - but is consistent with some JavaScript based encoders and decoders (as well - as Python's own decoder). If *allow_nan* is false, it will be a - :exc:`ValueError` to encode such floats. + ``JSON.stringify``. If *allow_nan* is true but not equal to ``'as_null'``, + then ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as + corresponding non-quote-delimited strings in the JSON output. This is the + default behavior. This behavior represents an extension of the JSON + specification, but is consistent with some JavaScript based encoders and + decoders (as well as Python's own decoder). If *allow_nan* is false, it + will be a :exc:`ValueError` to encode such floats. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that @@ -502,8 +502,8 @@ Encoders and Decoders All parameters are now :ref:`keyword-only `. .. versionchanged:: 3.14 - Added support for ``allow_nan='null'``. Passing any string value - other than ``'null'`` for ``allow_nan`` now raises a + Added support for ``allow_nan='as_null'``. Passing any string value + other than ``'as_null'`` for *allow_nan* now raises a :warning:`DeprecationWarning`. .. method:: default(o) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a43b0f30be6394..e69320e822ab3b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -761,14 +761,6 @@ itertools than the specified batch size. (Contributed by Raymond Hettinger in :gh:`113202`.) -json ----- - -* Add support for ``allow_nan='null'`` when encoding to JSON. This converts - floating-point infinities and NaNs to a JSON ``null``, for compatibility - with ECMAScript's ``JSON.stringify``. - (Contributed by Mark Dickinson in :gh:`115246`.) - marshal ------- diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bcb1098f43d5a3..6a3f9420c3379d 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -86,6 +86,14 @@ New Modules Improved Modules ================ +json +---- + +* Add support for ``allow_nan='as_null'`` when encoding to JSON. This converts + floating-point infinities and NaNs to a JSON ``null``, for compatibility + with ECMAScript's ``JSON.stringify``. + (Contributed by Mark Dickinson in :gh:`115246`.) + Optimizations ============= @@ -181,4 +189,3 @@ Deprecated Removed ------- - diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ca7355b2b61aa7..fde34ea21cdcc1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -800,6 +800,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(asend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(athrow)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fbb25285f0f282..35f97323c53ad6 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -289,6 +289,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) + STRUCT_FOR_ID(as_null) STRUCT_FOR_ID(asend) STRUCT_FOR_ID(ast) STRUCT_FOR_ID(athrow) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 508da40c53422d..4bf645ecac3c29 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -798,6 +798,7 @@ extern "C" { INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ + INIT_ID(as_null), \ INIT_ID(asend), \ INIT_ID(ast), \ INIT_ID(athrow), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cc2fc15ac5cabf..a97934ab53404a 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -708,6 +708,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(as_integer_ratio); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(as_null); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(asend); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index fd74b09e3a0275..f5b5ad33996544 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -150,10 +150,10 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, self.ensure_ascii = ensure_ascii self.check_circular = check_circular - if isinstance(allow_nan, str) and allow_nan != "null": + if isinstance(allow_nan, str) and allow_nan != 'as_null': warnings.warn( "in the future, allow_nan will no longer accept strings " - "other than 'null'. Use a boolean instead.", + "other than 'as_null'. Use a boolean instead.", DeprecationWarning, stacklevel=3, ) @@ -246,7 +246,7 @@ def floatstr(o, allow_nan=self.allow_nan, else: return _repr(o) - if allow_nan == 'null': + if allow_nan == 'as_null': return 'null' elif not allow_nan: diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index 34642a8371c11b..5279bd6d54dc44 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -35,29 +35,29 @@ def test_allow_nan(self): self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) def test_allow_nan_null(self): - # when allow_nan is "null", infinities and NaNs are converted to "null" + # when allow_nan is 'as_null', infinities and NaNs convert to 'null' for val in [float('inf'), float('-inf'), float('nan')]: with self.subTest(val=val): - out = self.dumps([val], allow_nan="null") + out = self.dumps([val], allow_nan='as_null') res = self.loads(out) self.assertEqual(res, [None]) # and finite values are treated as normal for val in [1.25, -23, -0.0, 0.0]: with self.subTest(val=val): - out = self.dumps([val], allow_nan="null") + out = self.dumps([val], allow_nan='as_null') res = self.loads(out) self.assertEqual(res, [val]) # testing a mixture vals = [-1.3, 1e100, -math.inf, 1234, -0.0, math.nan] - out = self.dumps(vals, allow_nan="null") + out = self.dumps(vals, allow_nan='as_null') res = self.loads(out) self.assertEqual(res, [-1.3, 1e100, None, 1234, -0.0, None]) def test_allow_nan_string_deprecation(self): with self.assertWarns(DeprecationWarning): - self.dumps(2.3, allow_nan="true") + self.dumps(2.3, allow_nan='true') def test_allow_nan_non_boolean(self): # check that exception gets propagated as expected diff --git a/Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst b/Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst index d752b1fe774cba..9bbee184cc8b8f 100644 --- a/Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst +++ b/Misc/NEWS.d/next/Library/2024-05-12-09-27-16.gh-issue-98306.dSlaEP.rst @@ -1,2 +1,2 @@ -Add support for ``allow_nan='null'`` when encoding an object to a JSON +Add support for ``allow_nan='as_null'`` when encoding an object to a JSON string. This converts floating-point infinities and NaNs to a JSON ``null``. diff --git a/Modules/_json.c b/Modules/_json.c index 80adbc78df05a1..7e0359912c47a9 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1229,7 +1229,8 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) // 0 to disallow nans and infinities // 1 to convert nans and infinities into corresponding JSON strings // 2 to convert nans and infinities to a JSON null - if (PyUnicode_Check(allow_nan_obj) && _PyUnicode_Equal(allow_nan_obj, &_Py_ID(null))) { + if (PyUnicode_Check(allow_nan_obj) && + _PyUnicode_Equal(allow_nan_obj, &_Py_ID(as_null))) { allow_nan = 2; } else { allow_nan = PyObject_IsTrue(allow_nan_obj); From ebcd36d347ec1dc47439b44516447e435b02a5ed Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 10:10:08 +0100 Subject: [PATCH 09/11] Documentation wording tweaks --- Doc/library/json.rst | 28 ++++++++++++++-------------- Doc/whatsnew/3.14.rst | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index fb5c891579a7c1..949f191edeff1b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -178,8 +178,8 @@ Basic Usage ``JSON.stringify``. If *allow_nan* is true but not equal to ``'as_null'`` then NaNs and infinities are converted to non-quote-delimited strings ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this - represents an extension of the JSON specification, and is not compliant with - standard JSON. + represents an extension of the JSON specification, and that the generated + output may not be accepted as valid JSON by third party JSON parsers. If *indent* is a non-negative integer or string, then JSON array elements and object members will be pretty-printed with that indent level. An indent level @@ -216,7 +216,7 @@ Basic Usage .. versionchanged:: 3.14 Added support for ``allow_nan='as_null'``. Passing any string value - other than ``'as_null'`` for ``allow_nan`` now raises a + other than ``'as_null'`` for ``allow_nan`` now triggers a :warning:`DeprecationWarning`. .. note:: @@ -225,7 +225,6 @@ Basic Usage so trying to serialize multiple objects with repeated calls to :func:`dump` using the same *fp* will result in an invalid JSON file. - .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ indent=None, separators=None, default=None, \ @@ -461,15 +460,16 @@ Encoders and Decoders prevent an infinite recursion (which would cause a :exc:`RecursionError`). Otherwise, no such check takes place. - If *allow_nan* is the string ``'as_null'``, then NaNs and infinities are - encoded as JSON ``null`` values. This matches the behavior of JavaScript's - ``JSON.stringify``. If *allow_nan* is true but not equal to ``'as_null'``, - then ``NaN``, ``Infinity``, and ``-Infinity`` will be encoded as - corresponding non-quote-delimited strings in the JSON output. This is the - default behavior. This behavior represents an extension of the JSON - specification, but is consistent with some JavaScript based encoders and - decoders (as well as Python's own decoder). If *allow_nan* is false, it - will be a :exc:`ValueError` to encode such floats. + If *allow_nan* is false (default: ``True``), then it will be a + :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, + ``inf``, ``-inf``) in strict compliance with the JSON specification. If + *allow_nan* is the string ``'as_null'``, NaNs and infinities will be + converted to a JSON ``null``, matching the behavior of JavaScript's + ``JSON.stringify``. If *allow_nan* is true but not equal to ``'as_null'`` + then NaNs and infinities are converted to non-quote-delimited strings + ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this + represents an extension of the JSON specification, and that the generated + output may not be accepted as valid JSON by third party JSON parsers. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that @@ -503,7 +503,7 @@ Encoders and Decoders .. versionchanged:: 3.14 Added support for ``allow_nan='as_null'``. Passing any string value - other than ``'as_null'`` for *allow_nan* now raises a + other than ``'as_null'`` for *allow_nan* now triggers a :warning:`DeprecationWarning`. .. method:: default(o) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 6a3f9420c3379d..3e302347657f6e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -90,7 +90,7 @@ json ---- * Add support for ``allow_nan='as_null'`` when encoding to JSON. This converts - floating-point infinities and NaNs to a JSON ``null``, for compatibility + floating-point infinities and NaNs to a JSON ``null``, for alignment with ECMAScript's ``JSON.stringify``. (Contributed by Mark Dickinson in :gh:`115246`.) From acf5642ee1dc3e962ef93fb291d1ab043d162716 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 10:21:47 +0100 Subject: [PATCH 10/11] Grammar and spacing tweaks --- Doc/library/json.rst | 6 +++--- Doc/whatsnew/3.14.rst | 1 + Lib/json/encoder.py | 3 --- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 949f191edeff1b..197e278e53e44f 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -179,7 +179,7 @@ Basic Usage then NaNs and infinities are converted to non-quote-delimited strings ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this represents an extension of the JSON specification, and that the generated - output may not be accepted as valid JSON by third party JSON parsers. + output may not be accepted as valid JSON by third-party JSON parsers. If *indent* is a non-negative integer or string, then JSON array elements and object members will be pretty-printed with that indent level. An indent level @@ -216,7 +216,7 @@ Basic Usage .. versionchanged:: 3.14 Added support for ``allow_nan='as_null'``. Passing any string value - other than ``'as_null'`` for ``allow_nan`` now triggers a + other than ``'as_null'`` for *allow_nan* now triggers a :warning:`DeprecationWarning`. .. note:: @@ -469,7 +469,7 @@ Encoders and Decoders then NaNs and infinities are converted to non-quote-delimited strings ``NaN``, ``Infinity`` and ``-Infinity`` in the JSON output. Note that this represents an extension of the JSON specification, and that the generated - output may not be accepted as valid JSON by third party JSON parsers. + output may not be accepted as valid JSON by third-party JSON parsers. If *sort_keys* is true (default: ``False``), then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3e302347657f6e..4fda5b31c81435 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -189,3 +189,4 @@ Deprecated Removed ------- + diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index f5b5ad33996544..382fba8aa97512 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -149,7 +149,6 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, self.skipkeys = skipkeys self.ensure_ascii = ensure_ascii self.check_circular = check_circular - if isinstance(allow_nan, str) and allow_nan != 'as_null': warnings.warn( "in the future, allow_nan will no longer accept strings " @@ -157,7 +156,6 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, DeprecationWarning, stacklevel=3, ) - self.allow_nan = allow_nan self.sort_keys = sort_keys self.indent = indent @@ -248,7 +246,6 @@ def floatstr(o, allow_nan=self.allow_nan, if allow_nan == 'as_null': return 'null' - elif not allow_nan: raise ValueError( "Out of range float values are not JSON compliant: " + From 3062f12c4a132fac05d142f47fcbdd8c123821ac Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 12 May 2024 10:30:39 +0100 Subject: [PATCH 11/11] Fix Sphinx role for DeprecationWarning --- Doc/library/json.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 197e278e53e44f..cc2013e5d7cefd 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -217,7 +217,7 @@ Basic Usage .. versionchanged:: 3.14 Added support for ``allow_nan='as_null'``. Passing any string value other than ``'as_null'`` for *allow_nan* now triggers a - :warning:`DeprecationWarning`. + :exc:`DeprecationWarning`. .. note:: @@ -504,7 +504,7 @@ Encoders and Decoders .. versionchanged:: 3.14 Added support for ``allow_nan='as_null'``. Passing any string value other than ``'as_null'`` for *allow_nan* now triggers a - :warning:`DeprecationWarning`. + :exc:`DeprecationWarning`. .. method:: default(o)