From dcba1b8b460e2af56367278a3d34d49a15d883be Mon Sep 17 00:00:00 2001 From: lubaskin Date: Tue, 26 Nov 2024 23:05:01 +0300 Subject: [PATCH 1/7] Add docs for token.EXACT_TOKEN_TYPES --- Doc/library/token.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 0cc9dddd91ed6b..4d51041da674dd 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,12 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + A dictionary that stores the string representation of the token + as keys, and their numeric codes as values. + + .. versionadded:: 3.8 + .. versionchanged:: 3.5 Added :data:`!AWAIT` and :data:`!ASYNC` tokens. From 931b175f0476c9ec9ed2642c7654692c62a957c3 Mon Sep 17 00:00:00 2001 From: lubaskin Date: Tue, 26 Nov 2024 23:14:37 +0300 Subject: [PATCH 2/7] first contribution --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index 08cd293eac3835..da764fa7148af7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1152,6 +1152,7 @@ Mike Lundy Zhongyue Luo Mark Lutz Taras Lyapun +Ilya Lyubavski Jim Lynch Mikael Lyngvig Jeff MacDonald From 1ddb3f9b5590a8014bca6f8abbaaccb8b9cc3644 Mon Sep 17 00:00:00 2001 From: lubaskin Date: Tue, 26 Nov 2024 23:17:35 +0300 Subject: [PATCH 3/7] fix bold text --- Doc/library/token.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 4d51041da674dd..831a0a364319d6 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -80,6 +80,7 @@ the :mod:`tokenize` module. .. data:: EXACT_TOKEN_TYPES + A dictionary that stores the string representation of the token as keys, and their numeric codes as values. From 7e19d69fe69272fc7add9718fac4400f6a4674a7 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 26 Nov 2024 17:20:49 -0500 Subject: [PATCH 4/7] Update Doc/library/token.rst Co-authored-by: Tomas R. --- Doc/library/token.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 831a0a364319d6..99798aecdd1fbe 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -81,8 +81,7 @@ the :mod:`tokenize` module. .. data:: EXACT_TOKEN_TYPES - A dictionary that stores the string representation of the token - as keys, and their numeric codes as values. + A dictionary mapping the string representation of a token to its numeric code. .. versionadded:: 3.8 From 1c7c39bb8aa5ce65d6a286fbca060408951dcaa2 Mon Sep 17 00:00:00 2001 From: lubaskin Date: Wed, 27 Nov 2024 16:12:07 +0300 Subject: [PATCH 5/7] add to __all__ and split --- Lib/token.py | 3 ++- .../Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst | 1 + Tools/build/generate_token.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst diff --git a/Lib/token.py b/Lib/token.py index b620317106e173..54d7cdccadc79a 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 00000000000000..7d95b97a59b124 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Expose :data:`token.EXACT_TOKEN_TYPES` in ``__all__``. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 16c38841e44a4d..d32747f19945d8 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'): # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d From 6f131ac5d0252a6bfb7cc8903de1e5102223ee3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=20=D0=9B=D1=8E=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D1=81=D0=BA=D0=B8=D0=B9?= <100635212+lubaskinc0de@users.noreply.github.com> Date: Wed, 27 Nov 2024 18:18:44 +0300 Subject: [PATCH 6/7] Update Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst index 7d95b97a59b124..58ebf5d0abe141 100644 --- a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -1 +1 @@ -Expose :data:`token.EXACT_TOKEN_TYPES` in ``__all__``. +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. From 723e05715d5669d479ded9e28a33b63ae8f85845 Mon Sep 17 00:00:00 2001 From: lubaskin Date: Wed, 27 Nov 2024 18:23:04 +0300 Subject: [PATCH 7/7] address review --- Doc/library/token.rst | 1 + Misc/ACKS | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 99798aecdd1fbe..40982f32b4beee 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -85,6 +85,7 @@ the :mod:`tokenize` module. .. versionadded:: 3.8 + .. versionchanged:: 3.5 Added :data:`!AWAIT` and :data:`!ASYNC` tokens. diff --git a/Misc/ACKS b/Misc/ACKS index da764fa7148af7..cd34846574b304 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1152,9 +1152,9 @@ Mike Lundy Zhongyue Luo Mark Lutz Taras Lyapun -Ilya Lyubavski Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre