diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e27a3b96d8fade..6d06ae8205dc01 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,13 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`AWAIT` and :data:`ASYNC` tokens. diff --git a/Lib/token.py b/Lib/token.py index 487f6edd3c951c..e26d36bd64e3c1 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Misc/ACKS b/Misc/ACKS index cb9de23eeeac11..0f8b79f26fbf6b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1139,6 +1139,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 00000000000000..58ebf5d0abe141 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 3bd307c1733867..6d27bc41ba2663 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'): # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d