From 5d5e187ba8015a0db94163e6f845aee4cfbd55cd Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 26 Jun 2025 17:46:10 +0000 Subject: [PATCH 1/2] Disallow explicit concat between Template and str --- Lib/test/test_tstring.py | 10 ++--- Objects/templateobject.c | 88 +++------------------------------------- Objects/unicodeobject.c | 15 ++----- 3 files changed, 13 insertions(+), 100 deletions(-) diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index aabae38556735b..7727f78498a3a0 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -161,9 +161,8 @@ def test_template_concatenation(self): # Test template + string t1 = t"Hello" - combined = t1 + ", world" - self.assertTStringEqual(combined, ("Hello, world",), ()) - self.assertEqual(fstring(combined), "Hello, world") + with self.assertRaises(TypeError): + _ = t1 + ", world" # Test template + template with interpolation name = "Python" @@ -174,9 +173,8 @@ def test_template_concatenation(self): self.assertEqual(fstring(combined), "Hello, Python") # Test string + template - t = "Hello, " + t"{name}" - self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) - self.assertEqual(fstring(t), "Hello, Python") + with self.assertRaises(TypeError): + _ = "Hello, " + t"{name}" def test_nested_templates(self): # Test a template inside another template expression diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 4293a311c440f7..0cc22d9986efc8 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -245,54 +245,6 @@ template_iter(PyObject *op) return (PyObject *)iter; } -static PyObject * -template_strings_append_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1); - PyObject *concat = PyUnicode_Concat(string, str); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - for (Py_ssize_t i = 0; i < stringslen - 1; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - PyTuple_SET_ITEM(newstrings, stringslen - 1, concat); - - return newstrings; -} - -static PyObject * -template_strings_prepend_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, 0); - PyObject *concat = PyUnicode_Concat(str, string); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - PyTuple_SET_ITEM(newstrings, 0, concat); - for (Py_ssize_t i = 1; i < stringslen; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - - return newstrings; -} - static PyObject * template_strings_concat(PyObject *left, PyObject *right) { @@ -344,46 +296,16 @@ template_concat_templates(templateobject *self, templateobject *other) return newtemplate; } -static PyObject * -template_concat_template_str(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_append_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - -static PyObject * -template_concat_str_template(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_prepend_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - PyObject * _PyTemplate_Concat(PyObject *self, PyObject *other) { if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) { return template_concat_templates((templateobject *) self, (templateobject *) other); - } - else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) { - return template_concat_template_str((templateobject *) self, other); - } - else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) { - return template_concat_str_template((templateobject *) other, self); - } - else { - Py_RETURN_NOTIMPLEMENTED; + } else { + PyErr_Format(PyExc_TypeError, + "can only concatenate Template (not \"%.200s\") to Template", + Py_TYPE(other)->tp_name); + return NULL; } } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5c2308a012142a..8df7a48284dccd 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_template.h" // _PyTemplate_Concat() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -11610,16 +11609,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right) return NULL; if (!PyUnicode_Check(right)) { - if (_PyTemplate_CheckExact(right)) { - // str + tstring is implemented in the tstring type - return _PyTemplate_Concat(left, right); - } - else { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - Py_TYPE(right)->tp_name); - return NULL; - } + PyErr_Format(PyExc_TypeError, + "can only concatenate str (not \"%.200s\") to str", + Py_TYPE(right)->tp_name); + return NULL; } /* Shortcuts */ From 1cc778f26a5236c68325d97e6b1cce4db1a24803 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 26 Jun 2025 18:17:12 +0000 Subject: [PATCH 2/2] First pass at removing implicit Template/str concat --- Lib/test/test_ast/test_ast.py | 7 ----- Lib/test/test_tstring.py | 51 +++++++++-------------------------- Lib/test/test_unparse.py | 4 --- Parser/action_helpers.c | 6 +++++ 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index cc46529c0ef105..4b84080cec9148 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -999,13 +999,6 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - # Test AST for implicit concat of t-string with f-string - tree = ast.parse('t"Hello {name}" f"{name}"') - self.assertIsInstance(tree.body[0].value, ast.TemplateStr) - self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) - self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue) - class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 7727f78498a3a0..b090276f966744 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -239,45 +239,20 @@ def test_literal_concatenation(self): self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) self.assertEqual(fstring(t), "Hello, Python") - # Test concatenation with string literal - name = "Python" - t = t"Hello, {name}" "and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with Unicode literal - name = "Python" - t = t"Hello, {name}" u"and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with f-string literal - tab = '\t' - t = t"Tab: {tab}. " f"f-tab: {tab}." - self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")]) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.") - - # Test concatenation with raw string literal - tab = '\t' - t = t"Tab: {tab}. " r"Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.") - - # Test concatenation with raw f-string literal - tab = '\t' - t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.") - + # Test disallowed mix of t-string and string what = 't' + expected_msg = 'cannot mix str and Template literals' + for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", + ): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) + + # Test disallowed mix of t-string and bytes expected_msg = 'cannot mix bytes and nonbytes literals' for case in ( "t'{what}-string literal' b'bytes literal'", diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d4db5e60af7978..0d6b05bc660b76 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -206,10 +206,6 @@ def test_tstrings(self): self.check_ast_roundtrip("t'foo'") self.check_ast_roundtrip("t'foo {bar}'") self.check_ast_roundtrip("t'foo {bar!s:.2f}'") - self.check_ast_roundtrip("t'foo {bar}' f'{bar}'") - self.check_ast_roundtrip("f'{bar}' t'foo {bar}'") - self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'") - self.check_ast_roundtrip("t'foo {bar}' u'bar'") def test_strings(self): self.check_ast_roundtrip("u'foo'") diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 0d362bf7a9111a..017e7984b95fdf 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1887,6 +1887,11 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, return NULL; } + if ((unicode_string_found || f_string_found) && t_string_found) { + RAISE_SYNTAX_ERROR("cannot mix str and Template literals"); + return NULL; + } + // If it's only bytes or only unicode string, do a simple concat if (!f_string_found && !t_string_found) { if (len == 1) { @@ -1902,6 +1907,7 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } } + if (t_string_found) { return _build_concatenated_template_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena);