Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.

nnorwitz · nnorwitz · commit db83eb3170eb · 2005-12-18T05:29:30.000Z
Needs backport.
diff --git a/Lib/test/bad_coding2.py b/Lib/test/bad_coding2.py
@@ -0,0 +1,2 @@
+﻿#coding: utf8
+print '我'
diff --git a/Lib/test/test_coding.py b/Lib/test/test_coding.py
@@ -5,6 +5,13 @@
 class CodingTest(unittest.TestCase):
     def test_bad_coding(self):
         module_name = 'bad_coding'
+        self.verify_bad_module(module_name)
+
+    def test_bad_coding2(self):
+        module_name = 'bad_coding2'
+        self.verify_bad_module(module_name)
+
+    def verify_bad_module(self, module_name):
         self.assertRaises(SyntaxError, __import__, 'test.' + module_name)
 
         path = os.path.dirname(__file__)
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -12,6 +12,8 @@ What's New in Python 2.5 alpha 1?
 Core and builtins
 -----------------
 
+- Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
+
 - Support for converting hex strings to floats no longer works.
   This was not portable.  float('0x3') now raises a ValueError.
 
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -292,6 +292,12 @@ check_coding_spec(const char* line, int size, struct tok_state *tok,
 			PyMem_DEL(cs);
 		}
 	}
+	if (!r) {
+		cs = tok->encoding;
+		if (!cs)
+			cs = "with BOM";
+		PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
+	}
 	return r;
 }
 
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -1439,8 +1439,8 @@ err_input(perrdetail *err)
 		}
 		if (msg == NULL)
 			msg = "unknown decode error";
-		Py_DECREF(type);
-		Py_DECREF(value);
+		Py_XDECREF(type);
+		Py_XDECREF(value);
 		Py_XDECREF(tb);
 		break;
 	}

Original file line number	Diff line number	Diff line change
`@@ -292,6 +292,12 @@ check_coding_spec(const char* line, int size, struct tok_state *tok,`
`292`	`292`	`PyMem_DEL(cs);`
`293`	`293`	`}`
`294`	`294`	`}`
	`295`	`+ if (!r) {`
	`296`	`+ cs = tok->encoding;`
	`297`	`+ if (!cs)`
	`298`	`+ cs = "with BOM";`
	`299`	`+ PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);`
	`300`	`+ }`
`295`	`301`	`return r;`
`296`	`302`	`}`
`297`	`303`
Original file line number	Diff line number	Diff line change
`@@ -1439,8 +1439,8 @@ err_input(perrdetail *err)`
`1439`	`1439`	`}`
`1440`	`1440`	`if (msg == NULL)`
`1441`	`1441`	`msg = "unknown decode error";`
`1442`		`- Py_DECREF(type);`
`1443`		`- Py_DECREF(value);`
	`1442`	`+ Py_XDECREF(type);`
	`1443`	`+ Py_XDECREF(value);`
`1444`	`1444`	`Py_XDECREF(tb);`
`1445`	`1445`	`break;`
`1446`	`1446`	`}`