Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis.

birkenfeld · birkenfeld · commit dde002899db8 · 2007-03-18T19:01:53.000Z
diff --git a/Grammar/Grammar b/Grammar/Grammar
@@ -107,7 +107,7 @@ power: atom trailer* ['**' factor]
 atom: ('(' [yield_expr|testlist_gexp] ')' |
        '[' [listmaker] ']' |
        '{' [dictsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '.' '.' '.')
+       NAME | NUMBER | STRING+ | '...')
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_gexp: test ( gen_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
diff --git a/Include/token.h b/Include/token.h
@@ -59,10 +59,11 @@ extern "C" {
 #define DOUBLESLASHEQUAL 49
 #define AT              50	
 #define RARROW          51
+#define ELLIPSIS        52
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP		52
-#define ERRORTOKEN	53
-#define N_TOKENS	54
+#define OP		53
+#define ERRORTOKEN	54
+#define N_TOKENS	55
 
 /* Special definitions for cooperation with parser */
 
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
@@ -113,7 +113,7 @@ def __init__(self):
                                token.LBRACE: self.atom_lbrace,
                                token.NUMBER: self.atom_number,
                                token.STRING: self.atom_string,
-                               token.DOT: self.atom_ellipsis,
+                               token.ELLIPSIS: self.atom_ellipsis,
                                token.NAME: self.atom_name,
                                }
         self.encoding = None
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
@@ -121,6 +121,7 @@ def testStringLiterals(self):
     def testEllipsis(self):
         x = ...
         self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
 
 class GrammarTests(unittest.TestCase):
 
diff --git a/Lib/token.py b/Lib/token.py
@@ -61,9 +61,10 @@
 DOUBLESLASHEQUAL = 49
 AT = 50
 RARROW = 51
-OP = 52
-ERRORTOKEN = 53
-N_TOKENS = 54
+ELLIPSIS = 52
+OP = 53
+ERRORTOKEN = 54
+N_TOKENS = 55
 NT_OFFSET = 256
 #--end constants--
 
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -83,7 +83,7 @@ def maybe(*choices): return group(*choices) + '?'
                  r"~")
 
 Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'[:;.,@]')
+Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
 Funny = group(Operator, Bracket, Special)
 
 PlainToken = group(Number, Funny, String, Name)
@@ -334,8 +334,8 @@ def generate_tokens(readline):
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
-                if initial in numchars or \
-                   (initial == '.' and token != '.'):      # ordinary number
+                if (initial in numchars or                  # ordinary number
+                    (initial == '.' and token != '.' and token != '...')):
                     yield (NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
                     yield (NL if parenlev > 0 else NEWLINE,
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -93,6 +93,7 @@ char *_PyParser_TokenNames[] = {
 	"DOUBLESLASHEQUAL",
 	"AT",
 	"RARROW",
+	"ELLIPSIS",
 	/* This table must match the #defines in token.h! */
 	"OP",
 	"<ERRORTOKEN>",
@@ -1082,6 +1083,16 @@ PyToken_ThreeChars(int c1, int c2, int c3)
 			break;
 		}
 		break;
+        case '.':
+	        switch (c2) {
+		case '.':
+			switch (c3) {
+			case '.':
+				return ELLIPSIS;
+			}
+			break;
+		}
+		break;
 	}
 	return OP;
 }
@@ -1278,13 +1289,22 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		c = tok_nextc(tok);
 		if (isdigit(c)) {
 			goto fraction;
-		}
-		else {
+		} else if (c == '.') {
+			c = tok_nextc(tok);
+			if (c == '.') {
+				*p_start = tok->start;
+				*p_end = tok->cur; 
+				return ELLIPSIS;
+			} else {
+				tok_backup(tok, c);
+			}
+			tok_backup(tok, '.');
+		} else {
 			tok_backup(tok, c);
-			*p_start = tok->start;
-			*p_end = tok->cur;
-			return DOT;
 		}
+		*p_start = tok->start;
+		*p_end = tok->cur;
+		return DOT;
 	}
 
 	/* Number */
diff --git a/Python/ast.c b/Python/ast.c
@@ -1410,7 +1410,7 @@ ast_for_atom(struct compiling *c, const node *n)
         PyArena_AddPyObject(c->c_arena, pynum);
         return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
     }
-    case DOT: /* Ellipsis */
+    case ELLIPSIS: /* Ellipsis */
         return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
     case LPAR: /* some parenthesized expressions */
         ch = CHILD(n, 1);
diff --git a/Python/graminit.c b/Python/graminit.c

Original file line number	Diff line number	Diff line change
`@@ -113,7 +113,7 @@ def __init__(self):`
`113`	`113`	`token.LBRACE: self.atom_lbrace,`
`114`	`114`	`token.NUMBER: self.atom_number,`
`115`	`115`	`token.STRING: self.atom_string,`
`116`		`- token.DOT: self.atom_ellipsis,`
	`116`	`+ token.ELLIPSIS: self.atom_ellipsis,`
`117`	`117`	`token.NAME: self.atom_name,`
`118`	`118`	`}`
`119`	`119`	`self.encoding = None`
Original file line number	Diff line number	Diff line change
`@@ -1410,7 +1410,7 @@ ast_for_atom(struct compiling c, const node n)`
`1410`	`1410`	`PyArena_AddPyObject(c->c_arena, pynum);`
`1411`	`1411`	`return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);`
`1412`	`1412`	`}`
`1413`		`- case DOT: /* Ellipsis */`
	`1413`	`+ case ELLIPSIS: /* Ellipsis */`
`1414`	`1414`	`return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);`
`1415`	`1415`	`case LPAR: /* some parenthesized expressions */`
`1416`	`1416`	`ch = CHILD(n, 1);`