From 356a58ffdabb078d46b196df91a9b8c37fb76184 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 3 Aug 2024 00:09:24 -0400 Subject: [PATCH 1/6] Fix available in admonitions --- docs/auxil/admonition_inserter.py | 7 ++++--- telegram/_payment/refundedpayment.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/auxil/admonition_inserter.py b/docs/auxil/admonition_inserter.py index 4227a845382..9455025331a 100644 --- a/docs/auxil/admonition_inserter.py +++ b/docs/auxil/admonition_inserter.py @@ -140,7 +140,7 @@ def _create_available_in(self) -> dict[type, str]: r"^\s*(?P[a-z_]+)" # Any number of spaces, named group for attribute r"\s?\(" # Optional whitespace, opening parenthesis r".*" # Any number of characters (that could denote a built-in type) - r":class:`.+`" # Marker of a classref, class name in backticks + r":(class|obj):`.+`" # Marker of a classref, class name in backticks r".*\):" # Any number of characters, closing parenthesis, colon. # The ^ colon above along with parenthesis is important because it makes sure that # the class is mentioned in the attribute description, not in free text. @@ -149,11 +149,11 @@ def _create_available_in(self) -> dict[type, str]: ) # for properties: there is no attr name in docstring. Just check if there's a class name. - prop_docstring_pattern = re.compile(r":class:`.+`.*:") + prop_docstring_pattern = re.compile(r":(class|obj):`.+`.*:") # pattern for iterating over potentially many class names in docstring for one attribute. # Tilde is optional (sometimes it is in the docstring, sometimes not). - single_class_name_pattern = re.compile(r":class:`~?(?P[\w.]*)`") + single_class_name_pattern = re.compile(r":(class|obj):`~?(?P[\w.]*)`") classes_to_inspect = inspect.getmembers(telegram, inspect.isclass) + inspect.getmembers( telegram.ext, inspect.isclass @@ -366,6 +366,7 @@ def _find_insert_pos_for_admonition(lines: list[str]) -> int: # to ".. admonition: Examples": ".. admonition:: Examples", ".. version", + "Args:", # The space after ":param" is important because docstring can contain # ":paramref:" in its plain text in the beginning of a line (e.g. ExtBot): ":param ", diff --git a/telegram/_payment/refundedpayment.py b/telegram/_payment/refundedpayment.py index 19bdfe84649..28d52226205 100644 --- a/telegram/_payment/refundedpayment.py +++ b/telegram/_payment/refundedpayment.py @@ -30,6 +30,8 @@ class RefundedPayment(TelegramObject): Objects of this class are comparable in terms of equality. Two objects of this class are considered equal, if their :attr:`telegram_payment_charge_id` is equal. + .. versionadded:: 21.4 + Args: currency (:obj:`str`): Three-letter ISO 4217 `currency `_ code, or ``XTR`` for From 42103f0c737ab303be06ffff1728851b405fa97b Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sun, 8 Sep 2024 00:36:56 -0400 Subject: [PATCH 2/6] WIP: Get basic idea working --- docs/auxil/admonition_inserter.py | 381 +++++++++++++++++------------- tests/docs/admonition_inserter.py | 10 +- 2 files changed, 222 insertions(+), 169 deletions(-) diff --git a/docs/auxil/admonition_inserter.py b/docs/auxil/admonition_inserter.py index 9455025331a..889a4161f5f 100644 --- a/docs/auxil/admonition_inserter.py +++ b/docs/auxil/admonition_inserter.py @@ -20,10 +20,24 @@ import re import typing from collections import defaultdict -from typing import Any, Iterator, Union +from typing import Any, Iterator, Literal, Union +from tests.auxil.slots import mro_slots import telegram import telegram.ext +import telegram.ext._utils.types +import telegram._utils.types +import telegram._utils.defaultvalue +from socket import socket +from apscheduler.job import Job as APSJob + +tg_objects = vars(telegram) +tg_objects.update(vars(telegram._utils.types)) +tg_objects.update(vars(telegram._utils.defaultvalue)) +tg_objects.update(vars(telegram.ext)) +tg_objects.update(vars(telegram.ext._utils.types)) +tg_objects.update(vars(telegram.ext._applicationbuilder)) +tg_objects.update({"socket": socket, "APSJob": APSJob}) def _iter_own_public_methods(cls: type) -> Iterator[tuple[str, type]]: @@ -49,6 +63,7 @@ class AdmonitionInserter: CLASS_ADMONITION_TYPES = ("use_in", "available_in", "returned_in") METHOD_ADMONITION_TYPES = ("shortcuts",) ALL_ADMONITION_TYPES = CLASS_ADMONITION_TYPES + METHOD_ADMONITION_TYPES + # ALL_ADMONITION_TYPES = ("use_in",) FORWARD_REF_PATTERN = re.compile(r"^ForwardRef\('(?P\w+)'\)$") """ A pattern to find a class name in a ForwardRef typing annotation. @@ -66,7 +81,7 @@ class AdmonitionInserter: METHOD_NAMES_FOR_BOT_AND_APPBUILDER: typing.ClassVar[dict[type, str]] = { cls: tuple(m[0] for m in _iter_own_public_methods(cls)) # m[0] means we take only names - for cls in (telegram.Bot, telegram.ext.ApplicationBuilder) + for cls in (telegram.Bot, telegram.ext.ApplicationBuilder, telegram.ext.Application) } """A dictionary mapping Bot and ApplicationBuilder classes to their relevant methods that will be mentioned in 'Returned in' and 'Use in' admonitions in other classes' docstrings. @@ -78,7 +93,11 @@ def __init__(self): # dynamically determine which method to use to create a sub-dictionary admonition_type: getattr(self, f"_create_{admonition_type}")() for admonition_type in self.ALL_ADMONITION_TYPES + # "use_in": self._create_use_in(), + # "shortcuts": self._create_shortcuts(), + # "available_in": self._create_available_in(), } + # print(self.admonitions["use_in"]) """Dictionary with admonitions. Contains sub-dictionaries, one per admonition type. Each sub-dictionary matches bot methods (for "Shortcuts") or telegram classes (for other admonition types) to texts of admonitions, e.g.: @@ -166,39 +185,47 @@ def _create_available_in(self) -> dict[type, str]: name_of_inspected_class_in_docstr = self._generate_class_name_for_link(inspected_class) # Parsing part of the docstring with attributes (parsing of properties follows later) - docstring_lines = inspect.getdoc(inspected_class).splitlines() - lines_with_attrs = [] - for idx, line in enumerate(docstring_lines): - if line.strip() == "Attributes:": - lines_with_attrs = docstring_lines[idx + 1 :] - break - - for line in lines_with_attrs: - if not (line_match := attr_docstr_pattern.match(line)): - continue - - target_attr = line_match.group("attr_name") - # a typing description of one attribute can contain multiple classes - for match in single_class_name_pattern.finditer(line): - name_of_class_in_attr = match.group("class_name") - - # Writing to dictionary: matching the class found in the docstring - # and its subclasses to the attribute of the class being inspected. - # The class in the attribute docstring (or its subclass) is the key, - # ReST link to attribute of the class currently being inspected is the value. - try: - self._resolve_arg_and_add_link( - arg=name_of_class_in_attr, - dict_of_methods_for_class=attrs_for_class, - link=f":attr:`{name_of_inspected_class_in_docstr}.{target_attr}`", - ) - except NotImplementedError as e: - raise NotImplementedError( - "Error generating Sphinx 'Available in' admonition " - f"(admonition_inserter.py). Class {name_of_class_in_attr} present in " - f"attribute {target_attr} of class {name_of_inspected_class_in_docstr}" - f" could not be resolved. {e!s}" - ) from e + # docstring_lines = inspect.getdoc(inspected_class).splitlines() + # lines_with_attrs = [] + # for idx, line in enumerate(docstring_lines): + # if line.strip() == "Attributes:": + # lines_with_attrs = docstring_lines[idx + 1 :] + # break + + # for line in lines_with_attrs: + # if not (line_match := attr_docstr_pattern.match(line)): + # continue + + # target_attr = line_match.group("attr_name") + # # a typing description of one attribute can contain multiple classes + # for match in single_class_name_pattern.finditer(line): + # name_of_class_in_attr = match.group("class_name") + + # Writing to dictionary: matching the class found in the docstring + # and its subclasses to the attribute of the class being inspected. + # The class in the attribute docstring (or its subclass) is the key, + # ReST link to attribute of the class currently being inspected is the value. + + # best effort - args of __init__ means not all attributes are covered, but there is no + # other way to get type hints of all attributes, other than doing ast parsing maybe. + # (Docstring parsing was discontinued with the closing of #4414) + type_hints = typing.get_type_hints(inspected_class.__init__, localns=tg_objects) + class_attrs = [slot for slot in mro_slots(inspected_class) if not slot.startswith("_")] + for target_attr in class_attrs: + try: + print(f"{inspected_class=},") + self._resolve_arg_and_add_link( + dict_of_methods_for_class=attrs_for_class, + link=f":attr:`{name_of_inspected_class_in_docstr}.{target_attr}`", + type_hints={target_attr: type_hints.get(target_attr)}, + ) + except NotImplementedError as e: + raise NotImplementedError( + "Error generating Sphinx 'Available in' admonition " + f"(admonition_inserter.py). Class {inspected_class} present in " + f"attribute {target_attr} of class {name_of_inspected_class_in_docstr}" + f" could not be resolved. {e!s}" + ) from e # Properties need to be parsed separately because they act like attributes but not # listed as attributes. @@ -209,39 +236,26 @@ def _create_available_in(self) -> dict[type, str]: if prop_name not in inspected_class.__dict__: continue - # 1. Can't use typing.get_type_hints because double-quoted type hints - # (like "Application") will throw a NameError - # 2. Can't use inspect.signature because return annotations of properties can be - # hard to parse (like "(self) -> BD"). - # 3. fget is used to access the actual function under the property wrapper - docstring = inspect.getdoc(getattr(inspected_class, prop_name).fget) - if docstring is None: - continue - - first_line = docstring.splitlines()[0] - if not prop_docstring_pattern.match(first_line): - continue + # fget is used to access the actual function under the property wrapper + type_hints = typing.get_type_hints(getattr(inspected_class, prop_name).fget, localns=tg_objects) - for match in single_class_name_pattern.finditer(first_line): - name_of_class_in_prop = match.group("class_name") - - # Writing to dictionary: matching the class found in the docstring and its - # subclasses to the property of the class being inspected. - # The class in the property docstring (or its subclass) is the key, - # ReST link to property of the class currently being inspected is the value. - try: - self._resolve_arg_and_add_link( - arg=name_of_class_in_prop, - dict_of_methods_for_class=attrs_for_class, - link=f":attr:`{name_of_inspected_class_in_docstr}.{prop_name}`", - ) - except NotImplementedError as e: - raise NotImplementedError( - "Error generating Sphinx 'Available in' admonition " - f"(admonition_inserter.py). Class {name_of_class_in_prop} present in " - f"property {prop_name} of class {name_of_inspected_class_in_docstr}" - f" could not be resolved. {e!s}" - ) from e + # Writing to dictionary: matching the class found in the docstring and its + # subclasses to the property of the class being inspected. + # The class in the property docstring (or its subclass) is the key, + # ReST link to property of the class currently being inspected is the value. + try: + self._resolve_arg_and_add_link( + dict_of_methods_for_class=attrs_for_class, + link=f":attr:`{name_of_inspected_class_in_docstr}.{prop_name}`", + type_hints={prop_name: type_hints.get(target_attr)}, + ) + except NotImplementedError as e: + raise NotImplementedError( + "Error generating Sphinx 'Available in' admonition " + f"(admonition_inserter.py). Class {inspected_class} present in " + f"property {prop_name} of class {name_of_inspected_class_in_docstr}" + f" could not be resolved. {e!s}" + ) from e return self._generate_admonitions(attrs_for_class, admonition_type="available_in") @@ -256,22 +270,22 @@ def _create_returned_in(self) -> dict[type, str]: for cls, method_names in self.METHOD_NAMES_FOR_BOT_AND_APPBUILDER.items(): for method_name in method_names: - sig = inspect.signature(getattr(cls, method_name)) - ret_annot = sig.return_annotation - method_link = self._generate_link_to_method(method_name, cls) + arg = getattr(cls, method_name) + print(arg, method_name) + ret_type_hint = typing.get_type_hints(arg, localns=tg_objects) try: self._resolve_arg_and_add_link( - arg=ret_annot, dict_of_methods_for_class=methods_for_class, link=method_link, + type_hints={"return": ret_type_hint.get("return")}, ) except NotImplementedError as e: raise NotImplementedError( "Error generating Sphinx 'Returned in' admonition " f"(admonition_inserter.py). {cls}, method {method_name}. " - f"Couldn't resolve type hint in return annotation {ret_annot}. {e!s}" + f"Couldn't resolve type hint in return annotation {ret_type_hint}. {e!s}" ) from e return self._generate_admonitions(methods_for_class, admonition_type="returned_in") @@ -330,22 +344,20 @@ def _create_use_in(self) -> dict[type, str]: for method_name in method_names: method_link = self._generate_link_to_method(method_name, cls) - sig = inspect.signature(getattr(cls, method_name)) - parameters = sig.parameters - - for param in parameters.values(): - try: - self._resolve_arg_and_add_link( - arg=param.annotation, - dict_of_methods_for_class=methods_for_class, - link=method_link, - ) - except NotImplementedError as e: - raise NotImplementedError( - "Error generating Sphinx 'Use in' admonition " - f"(admonition_inserter.py). {cls}, method {method_name}, parameter " - f"{param}: Couldn't resolve type hint {param.annotation}. {e!s}" - ) from e + arg = getattr(cls, method_name) + param_type_hints = typing.get_type_hints(arg, localns=tg_objects) + param_type_hints.pop("return", None) + try: + self._resolve_arg_and_add_link( + dict_of_methods_for_class=methods_for_class, + link=method_link, + type_hints=param_type_hints, + ) + except NotImplementedError as e: + raise NotImplementedError( + "Error generating Sphinx 'Use in' admonition " + f"(admonition_inserter.py). {cls}, method {method_name}, parameter " + ) from e return self._generate_admonitions(methods_for_class, admonition_type="use_in") @@ -448,6 +460,8 @@ def _generate_link_to_method(self, method_name: str, cls: type) -> str: @staticmethod def _iter_subclasses(cls: type) -> Iterator: + if not hasattr(cls, "__subclasses__") or cls is telegram.TelegramObject: + return iter([]) return ( # exclude private classes c @@ -457,9 +471,9 @@ def _iter_subclasses(cls: type) -> Iterator: def _resolve_arg_and_add_link( self, - arg: Any, dict_of_methods_for_class: defaultdict, link: str, + type_hints: dict[str, type], ) -> None: """A helper method. Tries to resolve the arg into a valid class. In case of success, adds the link (to a method, attribute, or property) for that class' and its subclasses' @@ -467,7 +481,9 @@ def _resolve_arg_and_add_link( **Modifies dictionary in place.** """ - for cls in self._resolve_arg(arg): + type_hints.pop("self", None) + + for cls in self._resolve_arg(type_hints): # When trying to resolve an argument from args or return annotation, # the method _resolve_arg returns None if nothing could be resolved. # Also, if class was resolved correctly, "telegram" will definitely be in its str(). @@ -479,88 +495,123 @@ def _resolve_arg_and_add_link( for subclass in self._iter_subclasses(cls): dict_of_methods_for_class[subclass].add(link) - def _resolve_arg(self, arg: Any) -> Iterator[Union[type, None]]: + def _resolve_arg(self, type_hints: dict[str, type]) -> list[type]: """Analyzes an argument of a method and recursively yields classes that the argument or its sub-arguments (in cases like Union[...]) belong to, if they can be resolved to telegram or telegram.ext classes. Raises `NotImplementedError`. """ - - origin = typing.get_origin(arg) - - if ( - origin in (collections.abc.Callable, typing.IO) - or arg is None - # no other check available (by type or origin) for these: - or str(type(arg)) in ("", "") - ): - pass - - # RECURSIVE CALLS - # for cases like Union[Sequence.... - elif origin in ( - Union, - collections.abc.Coroutine, - collections.abc.Sequence, - ): - for sub_arg in typing.get_args(arg): - yield from self._resolve_arg(sub_arg) - - elif isinstance(arg, typing.TypeVar): - # gets access to the "bound=..." parameter - yield from self._resolve_arg(arg.__bound__) - # END RECURSIVE CALLS - - elif isinstance(arg, typing.ForwardRef): - m = self.FORWARD_REF_PATTERN.match(str(arg)) - # We're sure it's a ForwardRef, so, unless it belongs to known exceptions, - # the class must be resolved. - # If it isn't resolved, we'll have the program throw an exception to be sure. - try: - cls = self._resolve_class(m.group("class_name")) - except AttributeError as exc: - # skip known ForwardRef's that need not be resolved to a Telegram class - if self.FORWARD_REF_SKIP_PATTERN.match(str(arg)): - pass - else: - raise NotImplementedError(f"Could not process ForwardRef: {arg}") from exc + telegram_classes = set() + + def recurse_type(typ): + if hasattr(typ, '__origin__'): # For generic types like Union, List, etc. + # Make sure it's not a telegram.ext generic type (e.g. ContextTypes[...]) + org = typing.get_origin(typ) + if "telegram.ext" in str(org): + telegram_classes.add(org) + + args = typing.get_args(typ) + # print(f"In recurse_type, found __origin__ {typ=}, {args=}") + for ar in args: + recurse_type(ar) + elif isinstance(typ, typing.TypeVar): + # gets access to the "bound=..." parameter + recurse_type(typ.__bound__) + elif inspect.isclass(typ) and "telegram" in inspect.getmodule(typ).__name__: + # print(f"typ is a class and inherits from TelegramObject: {typ=}") + telegram_classes.add(typ) else: - yield cls - - # For custom generics like telegram.ext._application.Application[~BT, ~CCT, ~UD...]. - # This must come before the check for isinstance(type) because GenericAlias can also be - # recognized as type if it belongs to . - elif str(type(arg)) in ( - "", - "", - "", - ): - if "telegram" in str(arg): - # get_origin() of telegram.ext._application.Application[~BT, ~CCT, ~UD...] - # will produce - yield origin - - elif isinstance(arg, type): - if "telegram" in str(arg): - yield arg - - # For some reason "InlineQueryResult", "InputMedia" & some others are currently not - # recognized as ForwardRefs and are identified as plain strings. - elif isinstance(arg, str): - # args like "ApplicationBuilder[BT, CCT, UD, CD, BD, JQ]" can be recognized as strings. - # Remove whatever is in the square brackets because it doesn't need to be parsed. - arg = re.sub(r"\[.+]", "", arg) - - cls = self._resolve_class(arg) - # Here we don't want an exception to be thrown since we're not sure it's ForwardRef - if cls is not None: - yield cls - - else: - raise NotImplementedError( - f"Cannot process argument {arg} of type {type(arg)} (origin {origin})" - ) + pass + # print(f"typ is not a class or doesn't inherit from TelegramObject: {typ=}. The " + # f"type is: {type(typ)=}") + # print(f"{inspect.isclass(typ)=}") + # if inspect.isclass(typ): + # print(f"{inspect.getmodule(typ).__name__=}") + + print() + print(f"in _resolve_arg {type_hints=}") + for param_name, type_hint in type_hints.items(): + print(f"{param_name=}", f"{type_hint=}") + if type_hint is None: + continue + recurse_type(type_hint) + print(f"{telegram_classes=}") + return list(telegram_classes) + # origin = typing.get_origin(arg) + + # if ( + # origin in (collections.abc.Callable, typing.IO) + # or arg is None + # # no other check available (by type or origin) for these: + # or str(type(arg)) in ("", "") + # ): + # pass + + # # RECURSIVE CALLS + # # for cases like Union[Sequence.... + # elif origin in ( + # Union, + # collections.abc.Coroutine, + # collections.abc.Sequence, + # ): + # for sub_arg in typing.get_args(arg): + # yield from self._resolve_arg(sub_arg) + + # elif isinstance(arg, typing.TypeVar): + # # gets access to the "bound=..." parameter + # yield from self._resolve_arg(arg.__bound__) + # # END RECURSIVE CALLS + + # elif isinstance(arg, typing.ForwardRef): + # m = self.FORWARD_REF_PATTERN.match(str(arg)) + # # We're sure it's a ForwardRef, so, unless it belongs to known exceptions, + # # the class must be resolved. + # # If it isn't resolved, we'll have the program throw an exception to be sure. + # try: + # cls = self._resolve_class(m.group("class_name")) + # except AttributeError as exc: + # # skip known ForwardRef's that need not be resolved to a Telegram class + # if self.FORWARD_REF_SKIP_PATTERN.match(str(arg)): + # pass + # else: + # raise NotImplementedError(f"Could not process ForwardRef: {arg}") from exc + # else: + # yield cls + + # # For custom generics like telegram.ext._application.Application[~BT, ~CCT, ~UD...]. + # # This must come before the check for isinstance(type) because GenericAlias can also be + # # recognized as type if it belongs to . + # elif str(type(arg)) in ( + # "", + # "", + # "", + # ): + # if "telegram" in str(arg): + # # get_origin() of telegram.ext._application.Application[~BT, ~CCT, ~UD...] + # # will produce + # yield origin + + # elif isinstance(arg, type): + # if "telegram" in str(arg): + # yield arg + + # # For some reason "InlineQueryResult", "InputMedia" & some others are currently not + # # recognized as ForwardRefs and are identified as plain strings. + # elif isinstance(arg, str): + # # args like "ApplicationBuilder[BT, CCT, UD, CD, BD, JQ]" can be recognized as strings. + # # Remove whatever is in the square brackets because it doesn't need to be parsed. + # arg = re.sub(r"\[.+]", "", arg) + + # cls = self._resolve_class(arg) + # # Here we don't want an exception to be thrown since we're not sure it's ForwardRef + # if cls is not None: + # yield cls + + # else: + # raise NotImplementedError( + # f"Cannot process argument {arg} of type {type(arg)} (origin {origin})" + # ) @staticmethod def _resolve_class(name: str) -> Union[type, None]: diff --git a/tests/docs/admonition_inserter.py b/tests/docs/admonition_inserter.py index fa19d9a0f9d..30a5494bf81 100644 --- a/tests/docs/admonition_inserter.py +++ b/tests/docs/admonition_inserter.py @@ -17,10 +17,12 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. -# This module is intentionally named without "test_" prefix. -# These tests are supposed to be run on GitHub when building docs. -# The tests require Python 3.9+ (just like AdmonitionInserter being tested), -# so they cannot be included in the main suite while older versions of Python are supported. +""" +This module is intentionally named without "test_" prefix. +These tests are supposed to be run on GitHub when building docs. +The tests require Python 3.10+ (just like AdmonitionInserter being tested), +so they cannot be included in the main suite while older versions of Python are supported. +""" import collections.abc From 85dd06b2f9edb4666da9de9925beb9985c9bb414 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler <22366557+Bibo-Joshi@users.noreply.github.com> Date: Fri, 24 Jan 2025 18:45:13 +0100 Subject: [PATCH 3/6] Apply some fixes and extend tests accordingly --- docs/auxil/admonition_inserter.py | 127 +++++++++++++++++------------- tests/docs/admonition_inserter.py | 21 ++++- 2 files changed, 93 insertions(+), 55 deletions(-) diff --git a/docs/auxil/admonition_inserter.py b/docs/auxil/admonition_inserter.py index 2e51655b456..c71a1eae3e1 100644 --- a/docs/auxil/admonition_inserter.py +++ b/docs/auxil/admonition_inserter.py @@ -21,23 +21,24 @@ import re import typing from collections import defaultdict +from collections.abc import Iterator +from socket import socket from types import FunctionType -from typing import Any, Iterator, Literal, Union, Generator -from tests.auxil.slots import mro_slots +from typing import Union + +from apscheduler.job import Job as APSJob import telegram +import telegram._utils.defaultvalue +import telegram._utils.types import telegram.ext - -# Some external & internal imports are necessary to make the symbols available during type resolution. import telegram.ext._utils.types -import telegram._utils.types -import telegram._utils.defaultvalue -from socket import socket -from apscheduler.job import Job as APSJob +from tests.auxil.slots import mro_slots # Define the namespace for type resolution. This helps dealing with the internal imports that # we do in many places -TG_NAMESPACE = vars(telegram) +# The .copy() is important to avoid modifying the original namespace +TG_NAMESPACE = vars(telegram).copy() TG_NAMESPACE.update(vars(telegram._utils.types)) TG_NAMESPACE.update(vars(telegram._utils.defaultvalue)) TG_NAMESPACE.update(vars(telegram.ext)) @@ -51,6 +52,17 @@ class PublicMethod(typing.NamedTuple): method: FunctionType +def _is_inherited_method(cls: type, method_name: str) -> bool: + """Checks if a method is inherited from a parent class. + Inheritance is not considered if the parent class is private. + Recurses through all direcot or indirect parent classes. + """ + # The [1:] slice is used to exclude the class itself from the MRO. + for base in cls.__mro__[1:]: + if method_name in base.__dict__ and not base.__name__.startswith("_"): + return True + return False + def _iter_own_public_methods(cls: type) -> Iterator[PublicMethod]: """Iterates over methods of a class that are not protected/private, @@ -60,12 +72,13 @@ def _iter_own_public_methods(cls: type) -> Iterator[PublicMethod]: This function is defined outside the class because it is used to create class constants. """ + # Use .isfunction() instead of .ismethod() because we want to include static methods. for m in inspect.getmembers(cls, predicate=inspect.isfunction): if ( not m[0].startswith("_") and m[0].islower() # to avoid camelCase methods - and m[0] in cls.__dict__ # method is not inherited from parent class + and not _is_inherited_method(cls, m[0]) ): yield PublicMethod(m[0], m[1]) @@ -76,7 +89,6 @@ class AdmonitionInserter: CLASS_ADMONITION_TYPES = ("use_in", "available_in", "returned_in") METHOD_ADMONITION_TYPES = ("shortcuts",) ALL_ADMONITION_TYPES = CLASS_ADMONITION_TYPES + METHOD_ADMONITION_TYPES - # ALL_ADMONITION_TYPES = ("use_in",) FORWARD_REF_PATTERN = re.compile(r"^ForwardRef\('(?P\w+)'\)$") """ A pattern to find a class name in a ForwardRef typing annotation. @@ -107,15 +119,11 @@ def __init__(self): # dynamically determine which method to use to create a sub-dictionary admonition_type: getattr(self, f"_create_{admonition_type}")() for admonition_type in self.ALL_ADMONITION_TYPES - # "use_in": self._create_use_in(), - # "shortcuts": self._create_shortcuts(), - # "available_in": self._create_available_in(), } - # print(self.admonitions["use_in"]) """Dictionary with admonitions. Contains sub-dictionaries, one per admonition type. Each sub-dictionary matches bot methods (for "Shortcuts") or telegram classes (for other admonition types) to texts of admonitions, e.g.: - + ``` { "use_in": { @@ -189,7 +197,6 @@ def _create_available_in(self) -> dict[type, str]: class_attrs = [slot for slot in mro_slots(inspected_class) if not slot.startswith("_")] for target_attr in class_attrs: try: - print(f"{inspected_class=},") self._resolve_arg_and_add_link( dict_of_methods_for_class=attrs_for_class, link=f":attr:`{name_of_inspected_class_in_docstr}.{target_attr}`", @@ -213,7 +220,9 @@ def _create_available_in(self) -> dict[type, str]: continue # fget is used to access the actual function under the property wrapper - type_hints = typing.get_type_hints(getattr(inspected_class, prop_name).fget, localns=TG_NAMESPACE) + type_hints = typing.get_type_hints( + getattr(inspected_class, prop_name).fget, localns=TG_NAMESPACE + ) # Writing to dictionary: matching the class found in the docstring and its # subclasses to the property of the class being inspected. @@ -239,7 +248,6 @@ def _create_returned_in(self) -> dict[type, str]: """Creates a dictionary with 'Returned in' admonitions for classes that are returned in Bot's and ApplicationBuilder's methods. """ - # Generate a mapping of classes to ReST links to Bot methods which return it, # i.e. {: {:meth:`telegram.Bot.send_message`, ...}} methods_for_class = defaultdict(set) @@ -248,7 +256,6 @@ def _create_returned_in(self) -> dict[type, str]: for method_name in method_names: method_link = self._generate_link_to_method(method_name, cls) arg = getattr(cls, method_name) - print(arg, method_name) ret_type_hint = typing.get_type_hints(arg, localns=TG_NAMESPACE) try: @@ -256,6 +263,7 @@ def _create_returned_in(self) -> dict[type, str]: dict_of_methods_for_class=methods_for_class, link=method_link, type_hints={"return": ret_type_hint.get("return")}, + resolve_nested_type_vars=False, ) except NotImplementedError as e: raise NotImplementedError( @@ -288,21 +296,23 @@ def _create_shortcuts(self) -> dict[collections.abc.Callable, str]: # inspect methods of all telegram classes for return statements that indicate # that this given method is a shortcut for a Bot method for _class_name, cls in inspect.getmembers(telegram, predicate=inspect.isclass): - # no need to inspect Bot's own methods, as Bot can't have shortcuts in Bot + if not cls.__module__.startswith("telegram"): + # For some reason inspect.getmembers() also yields some classes that are + # imported in the namespace but not part of the telegram module. + continue + if cls is telegram.Bot: + # no need to inspect Bot's own methods, as Bot can't have shortcuts in Bot continue for method_name, method in _iter_own_public_methods(cls): - print(f"{cls=}, {cls.__module__=}") # .getsourcelines() returns a tuple. Item [1] is an int for line in inspect.getsourcelines(method)[0]: if not (bot_method_match := bot_method_pattern.search(line)): continue bot_method = getattr(telegram.Bot, bot_method_match.group()) - link_to_shortcut_method = self._generate_link_to_method(method_name, cls) - shortcuts_for_bot_method[bot_method].add(link_to_shortcut_method) return self._generate_admonitions(shortcuts_for_bot_method, admonition_type="shortcuts") @@ -451,6 +461,7 @@ def _resolve_arg_and_add_link( dict_of_methods_for_class: defaultdict, link: str, type_hints: dict[str, type], + resolve_nested_type_vars: bool = True, ) -> None: """A helper method. Tries to resolve the arg into a valid class. In case of success, adds the link (to a method, attribute, or property) for that class' and its subclasses' @@ -460,7 +471,7 @@ def _resolve_arg_and_add_link( """ type_hints.pop("self", None) - for cls in self._resolve_arg(type_hints): + for cls in self._resolve_arg(type_hints, resolve_nested_type_vars): # When trying to resolve an argument from args or return annotation, # the method _resolve_arg returns None if nothing could be resolved. # Also, if class was resolved correctly, "telegram" will definitely be in its str(). @@ -472,48 +483,56 @@ def _resolve_arg_and_add_link( for subclass in self._iter_subclasses(cls): dict_of_methods_for_class[subclass].add(link) - def _resolve_arg(self, type_hints: dict[str, type]) -> list[type]: + def _resolve_arg( + self, type_hints: dict[str, type], resolve_nested_type_vars: bool + ) -> list[type]: """Analyzes an argument of a method and recursively yields classes that the argument or its sub-arguments (in cases like Union[...]) belong to, if they can be resolved to telegram or telegram.ext classes. + Args: + type_hints: A dictionary of argument names and their types. + resolve_nested_type_vars: If True, nested type variables (like Application[BT, …]) + will be resolved to their actual classes. If False, only the outermost type + variable will be resolved. *Only* affects ptb classes, not built-in types. + Useful for checking the return type of methods, where nested type variables + are not really useful. + Raises `NotImplementedError`. """ + + def _is_ptb_class(cls: type) -> bool: + if not hasattr(cls, "__module__"): + return False + return cls.__module__.startswith("telegram") + + # will be edited in place telegram_classes = set() - def recurse_type(typ): - if hasattr(typ, '__origin__'): # For generic types like Union, List, etc. + def recurse_type(type_, is_recursed_from_ptb_class: bool): + next_is_recursed_from_ptb_class = is_recursed_from_ptb_class or _is_ptb_class(type_) + + if hasattr(type_, "__origin__"): # For generic types like Union, List, etc. # Make sure it's not a telegram.ext generic type (e.g. ContextTypes[...]) - org = typing.get_origin(typ) + org = typing.get_origin(type_) if "telegram.ext" in str(org): telegram_classes.add(org) - args = typing.get_args(typ) - # print(f"In recurse_type, found __origin__ {typ=}, {args=}") - for ar in args: - recurse_type(ar) - elif isinstance(typ, typing.TypeVar): + args = typing.get_args(type_) + for arg in args: + recurse_type(arg, next_is_recursed_from_ptb_class) + elif isinstance(type_, typing.TypeVar) and ( + resolve_nested_type_vars or not is_recursed_from_ptb_class + ): # gets access to the "bound=..." parameter - recurse_type(typ.__bound__) - elif inspect.isclass(typ) and "telegram" in inspect.getmodule(typ).__name__: - # print(f"typ is a class and inherits from TelegramObject: {typ=}") - telegram_classes.add(typ) - else: - pass - # print(f"typ is not a class or doesn't inherit from TelegramObject: {typ=}. The " - # f"type is: {type(typ)=}") - # print(f"{inspect.isclass(typ)=}") - # if inspect.isclass(typ): - # print(f"{inspect.getmodule(typ).__name__=}") - - print() - print(f"in _resolve_arg {type_hints=}") - for param_name, type_hint in type_hints.items(): - print(f"{param_name=}", f"{type_hint=}") - if type_hint is None: - continue - recurse_type(type_hint) - print(f"{telegram_classes=}") + recurse_type(type_.__bound__, next_is_recursed_from_ptb_class) + elif inspect.isclass(type_) and "telegram" in inspect.getmodule(type_).__name__: + telegram_classes.add(type_) + + for type_hint in type_hints.values(): + if type_hint is not None: + recurse_type(type_hint, False) + return list(telegram_classes) @staticmethod diff --git a/tests/docs/admonition_inserter.py b/tests/docs/admonition_inserter.py index ad40f96c884..d586cdff363 100644 --- a/tests/docs/admonition_inserter.py +++ b/tests/docs/admonition_inserter.py @@ -142,6 +142,18 @@ def test_admonitions_dict(self, admonition_inserter): # one of which is with Bot ":meth:`telegram.CallbackQuery.edit_message_caption`", ), + ( + "shortcuts", + telegram.Bot.ban_chat_member, + # ban_member is defined on the private parent class _ChatBase + ":meth:`telegram.Chat.ban_member`", + ), + ( + "shortcuts", + telegram.Bot.ban_chat_member, + # ban_member is defined on the private parent class _ChatBase + ":meth:`telegram.ChatFullInfo.ban_member`", + ), ( "use_in", telegram.InlineQueryResult, @@ -212,9 +224,16 @@ def test_check_presence(self, admonition_inserter, admonition_type, cls, link): "returned_in", telegram.ext.CallbackContext, # -> Application[BT, CCT, UD, CD, BD, JQ]. - # In this case classes inside square brackets must not be parsed + # The type vars are not really part of the return value, so we don't expect them ":meth:`telegram.ext.ApplicationBuilder.build`", ), + ( + "returned_in", + telegram.Bot, + # -> Application[BT, CCT, UD, CD, BD, JQ]. + # The type vars are not really part of the return value, so we don't expect them + ":meth:`telegram.ext.ApplicationBuilder.bot`", + ), ], ) def test_check_absence(self, admonition_inserter, admonition_type, cls, link): From 57b3015b22c8ba491dc35b38aafa258b4c4d3449 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler <22366557+Bibo-Joshi@users.noreply.github.com> Date: Fri, 24 Jan 2025 19:02:12 +0100 Subject: [PATCH 4/6] Small CSS improvement --- docs/source/_static/style_admonitions.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_static/style_admonitions.css b/docs/source/_static/style_admonitions.css index 89c0d4b9e5e..4d86486afe9 100644 --- a/docs/source/_static/style_admonitions.css +++ b/docs/source/_static/style_admonitions.css @@ -61,5 +61,5 @@ } .admonition.returned-in > ul, .admonition.available-in > ul, .admonition.use-in > ul, .admonition.shortcuts > ul { max-height: 200px; - overflow-y: scroll; + overflow-y: auto; } From c5c6d441d65a36ea140ead3da4e376634c32d7ac Mon Sep 17 00:00:00 2001 From: Hinrich Mahler <22366557+Bibo-Joshi@users.noreply.github.com> Date: Sat, 25 Jan 2025 23:03:05 +0100 Subject: [PATCH 5/6] Extend available-in for properties --- docs/auxil/admonition_inserter.py | 19 +++++++++++-------- tests/docs/admonition_inserter.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/auxil/admonition_inserter.py b/docs/auxil/admonition_inserter.py index c71a1eae3e1..cba9f3575fc 100644 --- a/docs/auxil/admonition_inserter.py +++ b/docs/auxil/admonition_inserter.py @@ -97,13 +97,6 @@ class AdmonitionInserter: start and end markers. """ - FORWARD_REF_SKIP_PATTERN = re.compile(r"^ForwardRef\('DefaultValue\[\w+]'\)$") - """A pattern that will be used to skip known ForwardRef's that need not be resolved - to a Telegram class, e.g.: - ForwardRef('DefaultValue[None]') - ForwardRef('DefaultValue[DVValueType]') - """ - METHOD_NAMES_FOR_BOT_APP_APPBUILDER: typing.ClassVar[dict[type, str]] = { cls: tuple(m.name for m in _iter_own_public_methods(cls)) for cls in (telegram.Bot, telegram.ext.ApplicationBuilder, telegram.ext.Application) @@ -201,6 +194,7 @@ def _create_available_in(self) -> dict[type, str]: dict_of_methods_for_class=attrs_for_class, link=f":attr:`{name_of_inspected_class_in_docstr}.{target_attr}`", type_hints={target_attr: type_hints.get(target_attr)}, + resolve_nested_type_vars=False, ) except NotImplementedError as e: raise NotImplementedError( @@ -232,7 +226,8 @@ def _create_available_in(self) -> dict[type, str]: self._resolve_arg_and_add_link( dict_of_methods_for_class=attrs_for_class, link=f":attr:`{name_of_inspected_class_in_docstr}.{prop_name}`", - type_hints={prop_name: type_hints.get(target_attr)}, + type_hints={prop_name: type_hints.get("return")}, + resolve_nested_type_vars=False, ) except NotImplementedError as e: raise NotImplementedError( @@ -528,6 +523,14 @@ def recurse_type(type_, is_recursed_from_ptb_class: bool): recurse_type(type_.__bound__, next_is_recursed_from_ptb_class) elif inspect.isclass(type_) and "telegram" in inspect.getmodule(type_).__name__: telegram_classes.add(type_) + elif isinstance(type_, typing.ForwardRef): + # Resolving ForwardRef is not easy. https://peps.python.org/pep-0749/ will + # hopefully make it better by introducing typing.resolve_forward_ref() in py3.14 + # but that's not there yet + # So for now we fall back to a best effort approach of guessing if the class is + # available in tg or tg.ext + with contextlib.suppress(AttributeError): + telegram_classes.add(self._resolve_class(type_.__forward_arg__)) for type_hint in type_hints.values(): if type_hint is not None: diff --git a/tests/docs/admonition_inserter.py b/tests/docs/admonition_inserter.py index d586cdff363..9d96cfd3e9f 100644 --- a/tests/docs/admonition_inserter.py +++ b/tests/docs/admonition_inserter.py @@ -105,6 +105,21 @@ def test_admonitions_dict(self, admonition_inserter): telegram.ResidentialAddress, # mentioned on the second line of docstring of .data ":attr:`telegram.EncryptedPassportElement.data`", ), + ( + "available_in", + telegram.ext.JobQueue, + ":attr:`telegram.ext.CallbackContext.job_queue`", + ), + ( + "available_in", + telegram.ext.Application, + ":attr:`telegram.ext.CallbackContext.application`", + ), + ( + "available_in", + telegram.Bot, + ":attr:`telegram.ext.CallbackContext.bot`", + ), ( "returned_in", telegram.StickerSet, From 4069fafceb9136547246469a72b609d846738675 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler <22366557+Bibo-Joshi@users.noreply.github.com> Date: Sat, 25 Jan 2025 23:14:33 +0100 Subject: [PATCH 6/6] Fix bug in Application Slots --- docs/auxil/admonition_inserter.py | 4 +++- telegram/ext/_application.py | 6 ++---- tests/docs/admonition_inserter.py | 5 +++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/auxil/admonition_inserter.py b/docs/auxil/admonition_inserter.py index cba9f3575fc..56d63d08cb2 100644 --- a/docs/auxil/admonition_inserter.py +++ b/docs/auxil/admonition_inserter.py @@ -479,7 +479,9 @@ def _resolve_arg_and_add_link( dict_of_methods_for_class[subclass].add(link) def _resolve_arg( - self, type_hints: dict[str, type], resolve_nested_type_vars: bool + self, + type_hints: dict[str, type], + resolve_nested_type_vars: bool, ) -> list[type]: """Analyzes an argument of a method and recursively yields classes that the argument or its sub-arguments (in cases like Union[...]) belong to, if they can be resolved to diff --git a/telegram/ext/_application.py b/telegram/ext/_application.py index 5815b34a2eb..883c475ed76 100644 --- a/telegram/ext/_application.py +++ b/telegram/ext/_application.py @@ -235,7 +235,7 @@ class Application( """ __slots__ = ( - ( # noqa: RUF005 + ( "__create_task_tasks", "__update_fetcher_task", "__update_persistence_event", @@ -270,9 +270,7 @@ class Application( # Allowing '__weakref__' creation here since we need it for the JobQueue # Currently the __weakref__ slot is already created # in the AsyncContextManager base class for pythons < 3.13 - + ("__weakref__",) - if sys.version_info >= (3, 13) - else () + + (("__weakref__",) if sys.version_info >= (3, 13) else ()) ) def __init__( diff --git a/tests/docs/admonition_inserter.py b/tests/docs/admonition_inserter.py index 9d96cfd3e9f..76c40189699 100644 --- a/tests/docs/admonition_inserter.py +++ b/tests/docs/admonition_inserter.py @@ -120,6 +120,11 @@ def test_admonitions_dict(self, admonition_inserter): telegram.Bot, ":attr:`telegram.ext.CallbackContext.bot`", ), + ( + "available_in", + telegram.Bot, + ":attr:`telegram.ext.Application.bot`", + ), ( "returned_in", telegram.StickerSet,