Issue 21977: Minor improvements to the regexes in the tokenizer example.

rhettinger · rhettinger · commit 8323f68f3e50 · 2014-07-14T01:52:00.000-07:00
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
@@ -1340,9 +1340,9 @@ successive matches::
             ('ASSIGN',  r':='),          # Assignment operator
             ('END',     r';'),           # Statement terminator
             ('ID',      r'[A-Za-z]+'),   # Identifiers
-            ('OP',      r'[+*\/\-]'),    # Arithmetic operators
+            ('OP',      r'[+\-*/]'),     # Arithmetic operators
             ('NEWLINE', r'\n'),          # Line endings
-            ('SKIP',    r'[ \t]'),       # Skip over spaces and tabs
+            ('SKIP',    r'[ \t]+'),      # Skip over spaces and tabs
         ]
         tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification)
         get_token = re.compile(tok_regex).match
