Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 025eb98

Browse files
ammaraskargvanrossum
ammaraskar
authored and
gvanrossum
committed
bpo-34683: Make SyntaxError column offsets consistently 1-indexed (gh-9338)
Also point to start of tokens in parsing errors. Fixes bpo-34683
1 parent 223e501 commit 025eb98

11 files changed

Lines changed: 65 additions & 21 deletions

File tree

‎Lib/test/test_exceptions.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,45 @@ def check(src, lineno, offset):
187187
check('def spam():\n print(1)\n print(2)', 3, 10)
188188
check('Python = "Python" +', 1, 20)
189189
check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
190+
check('x = "a', 1, 7)
191+
check('lambda x: x = 2', 1, 1)
192+
193+
# Errors thrown by compile.c
194+
check('class foo:return 1', 1, 11)
195+
check('def f():\n continue', 2, 3)
196+
check('def f():\n break', 2, 3)
197+
check('try:\n pass\nexcept:\n pass\nexcept ValueError:\n pass', 2, 3)
198+
199+
# Errors thrown by tokenizer.c
200+
check('(0x+1)', 1, 3)
201+
check('x = 0xI', 1, 6)
202+
check('0010 + 2', 1, 4)
203+
check('x = 32e-+4', 1, 8)
204+
check('x = 0o9', 1, 6)
205+
206+
# Errors thrown by symtable.c
207+
check('x = [(yield i) for i in range(3)]', 1, 6)
208+
check('def f():\n from _ import *', 1, 1)
209+
check('def f(x, x):\n pass', 1, 1)
210+
check('def f(x):\n nonlocal x', 2, 3)
211+
check('def f(x):\n x = 1\n global x', 3, 3)
212+
check('nonlocal x', 1, 1)
213+
check('def f():\n global x\n nonlocal x', 2, 3)
214+
215+
# Errors thrown by ast.c
216+
check('for 1 in []: pass', 1, 5)
217+
check('def f(*):\n pass', 1, 7)
218+
check('[*x for x in xs]', 1, 2)
219+
check('def f():\n x, y: int', 2, 3)
220+
check('(yield i) = 2', 1, 1)
221+
check('foo(x for x in range(10), 100)', 1, 5)
222+
check('foo(1=2)', 1, 5)
223+
224+
# Errors thrown by future.c
225+
check('from __future__ import doesnt_exist', 1, 1)
226+
check('from __future__ import braces', 1, 1)
227+
check('x=1\nfrom __future__ import division', 2, 1)
228+
190229

191230
@cpython_only
192231
def testSettingException(self):

‎Lib/test/test_future.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def get_error_location(msg):
1515

1616
class FutureTest(unittest.TestCase):
1717

18-
def check_syntax_error(self, err, basename, lineno, offset=0):
18+
def check_syntax_error(self, err, basename, lineno, offset=1):
1919
self.assertIn('%s.py, line %d' % (basename, lineno), str(err))
2020
self.assertEqual(os.path.basename(err.filename), basename + '.py')
2121
self.assertEqual(err.lineno, lineno)
@@ -68,12 +68,12 @@ def test_badfuture8(self):
6868
def test_badfuture9(self):
6969
with self.assertRaises(SyntaxError) as cm:
7070
from test import badsyntax_future9
71-
self.check_syntax_error(cm.exception, "badsyntax_future9", 3, 0)
71+
self.check_syntax_error(cm.exception, "badsyntax_future9", 3)
7272

7373
def test_badfuture10(self):
7474
with self.assertRaises(SyntaxError) as cm:
7575
from test import badsyntax_future10
76-
self.check_syntax_error(cm.exception, "badsyntax_future10", 3, 0)
76+
self.check_syntax_error(cm.exception, "badsyntax_future10", 3)
7777

7878
def test_parserhack(self):
7979
# test that the parser.c::future_hack function works as expected

‎Lib/test/test_global.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ def wrong1():
2424
global a
2525
global b
2626
"""
27-
check_syntax_error(self, prog_text_1, lineno=4, offset=4)
27+
check_syntax_error(self, prog_text_1, lineno=4, offset=5)
2828

2929
def test2(self):
3030
prog_text_2 = """\
3131
def wrong2():
3232
print(x)
3333
global x
3434
"""
35-
check_syntax_error(self, prog_text_2, lineno=3, offset=4)
35+
check_syntax_error(self, prog_text_2, lineno=3, offset=5)
3636

3737
def test3(self):
3838
prog_text_3 = """\
@@ -41,7 +41,7 @@ def wrong3():
4141
x = 2
4242
global x
4343
"""
44-
check_syntax_error(self, prog_text_3, lineno=4, offset=4)
44+
check_syntax_error(self, prog_text_3, lineno=4, offset=5)
4545

4646
def test4(self):
4747
prog_text_4 = """\

‎Lib/test/test_support.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ def test_make_bad_fd(self):
286286
self.assertEqual(cm.exception.errno, errno.EBADF)
287287

288288
def test_check_syntax_error(self):
289-
support.check_syntax_error(self, "def class", lineno=1, offset=9)
289+
support.check_syntax_error(self, "def class", lineno=1, offset=5)
290290
with self.assertRaises(AssertionError):
291291
support.check_syntax_error(self, "x=1")
292292

‎Lib/test/test_symtable.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ def checkfilename(brokencode, offset):
169169
else:
170170
self.fail("no SyntaxError for %r" % (brokencode,))
171171
checkfilename("def f(x): foo)(", 14) # parse-time
172-
checkfilename("def f(x): global x", 10) # symtable-build-time
172+
checkfilename("def f(x): global x", 11) # symtable-build-time
173173
symtable.symtable("pass", b"spam", "exec")
174174
with self.assertWarns(DeprecationWarning), \
175175
self.assertRaises(TypeError):

‎Misc/NEWS.d/next/Core and Builtins/2018-09-15-19-32-34.bpo-34683.msCiQE.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed a bug where some SyntaxError error pointed to locations that were off-by-one.

‎Parser/parsetok.c‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
187187
parser_state *ps;
188188
node *n;
189189
int started = 0;
190+
int col_offset;
190191

191192
if ((ps = PyParser_New(g, start)) == NULL) {
192193
err_ret->error = E_NOMEM;
@@ -203,7 +204,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
203204
int type;
204205
size_t len;
205206
char *str;
206-
int col_offset;
207+
col_offset = -1;
207208

208209
type = PyTokenizer_Get(tok, &a, &b);
209210
if (type == ERRORTOKEN) {
@@ -321,7 +322,10 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
321322
if (tok->buf != NULL) {
322323
size_t len;
323324
assert(tok->cur - tok->buf < INT_MAX);
324-
err_ret->offset = (int)(tok->cur - tok->buf);
325+
/* if we've managed to parse a token, point the offset to its start,
326+
* else use the current reading position of the tokenizer
327+
*/
328+
err_ret->offset = col_offset != -1 ? col_offset + 1 : ((int)(tok->cur - tok->buf));
325329
len = tok->inp - tok->buf;
326330
err_ret->text = (char *) PyObject_MALLOC(len + 1);
327331
if (err_ret->text != NULL) {

‎Python/ast.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ ast_error(struct compiling *c, const node *n, const char *errmsg)
683683
Py_INCREF(Py_None);
684684
loc = Py_None;
685685
}
686-
tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset, loc);
686+
tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset + 1, loc);
687687
if (!tmp)
688688
return 0;
689689
errstr = PyUnicode_FromString(errmsg);

‎Python/compile.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4830,7 +4830,7 @@ compiler_error(struct compiler *c, const char *errstr)
48304830
loc = Py_None;
48314831
}
48324832
u = Py_BuildValue("(OiiO)", c->c_filename, c->u->u_lineno,
4833-
c->u->u_col_offset, loc);
4833+
c->u->u_col_offset + 1, loc);
48344834
if (!u)
48354835
goto exit;
48364836
v = Py_BuildValue("(zO)", errstr, u);

‎Python/future.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename)
4848
} else if (strcmp(feature, "braces") == 0) {
4949
PyErr_SetString(PyExc_SyntaxError,
5050
"not a chance");
51-
PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset);
51+
PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
5252
return 0;
5353
} else {
5454
PyErr_Format(PyExc_SyntaxError,
5555
UNDEFINED_FUTURE_FEATURE, feature);
56-
PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset);
56+
PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
5757
return 0;
5858
}
5959
}

0 commit comments

Comments
 (0)