From 29f10c6b29d658a9b56a4a2f7093d321b55f0df7 Mon Sep 17 00:00:00 2001 From: bastimeyer Date: Mon, 29 Jan 2024 20:08:24 +0100 Subject: [PATCH 1/3] utils.module: fix load_module(), add exec_module() - Use global finder cache via `pkgutil.get_importer()` instead of always creating a new `FileFinder` object for each module - Add `exec_module()` for being able to reuse `PathEntryFinder` objects returned by `pkgutil.iter_modules()` - Add support for pathlib path objects - Update tests --- src/streamlink/utils/module.py | 25 +++++++++++++++---- tests/utils/test_module.py | 44 +++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/streamlink/utils/module.py b/src/streamlink/utils/module.py index 50e0fa7d985..850c58c81c0 100644 --- a/src/streamlink/utils/module.py +++ b/src/streamlink/utils/module.py @@ -1,15 +1,30 @@ -from importlib.machinery import SOURCE_SUFFIXES, FileFinder, SourceFileLoader +from importlib.abc import PathEntryFinder +from importlib.machinery import FileFinder from importlib.util import module_from_spec +from pathlib import Path +from pkgutil import get_importer +from types import ModuleType +from typing import Union -_loader_details = [(SourceFileLoader, SOURCE_SUFFIXES)] +def load_module(name: str, path: Union[Path, str]) -> ModuleType: + path = str(path) + finder = get_importer(path) + if not finder: + raise ImportError(f"Not a package path: {path}", path=path) + return exec_module(finder, name) -def load_module(name, path=None): - finder = FileFinder(path, *_loader_details) + +def exec_module(finder: PathEntryFinder, name: str) -> ModuleType: spec = finder.find_spec(name) if not spec or not spec.loader: - raise ImportError(f"no module named {name}") + raise ImportError( + f"No module named '{name}'", + name=name, + path=finder.path if isinstance(finder, FileFinder) else None, + ) mod = module_from_spec(spec) spec.loader.exec_module(mod) + return mod diff --git a/tests/utils/test_module.py b/tests/utils/test_module.py index fddde8eb7d0..3b01d418c2b 100644 --- a/tests/utils/test_module.py +++ b/tests/utils/test_module.py @@ -1,5 +1,5 @@ -import os.path import sys +from pathlib import Path import pytest @@ -9,12 +9,40 @@ # used in the import test to verify that this module was imported __test_marker__ = "test_marker" +_here = Path(__file__).parent -class TestUtilsModule: - def test_load_module_non_existent(self): - with pytest.raises(ImportError): - load_module("non_existent_module", os.path.dirname(__file__)) - def test_load_module(self): - assert load_module(__name__.split(".")[-1], os.path.dirname(__file__)).__test_marker__ \ - == sys.modules[__name__].__test_marker__ +@pytest.mark.parametrize(("name", "path", "expected"), [ + pytest.param( + "some_module", + _here / "does_not_exist", + ImportError( + f"Not a package path: {_here / 'does_not_exist'}", + path=str(_here / "does_not_exist"), + ), + id="no-package", + ), + pytest.param( + "does_not_exist", + _here, + ImportError( + "No module named 'does_not_exist'", + name="does_not_exist", + path=str(_here), + ), + id="no-module", + ), +]) +def test_load_module_importerror(name: str, path: Path, expected: ImportError): + with pytest.raises(ImportError) as cm: + load_module(name, path) + assert cm.value.msg == expected.msg + assert cm.value.name == expected.name + assert cm.value.path == expected.path + + +def test_load_module(): + mod = load_module(__name__.split(".")[-1], Path(__file__).parent) + assert "__test_marker__" in mod.__dict__ + assert mod is not sys.modules[__name__] + assert mod.__test_marker__ == sys.modules[__name__].__test_marker__ From 355309473eb867c99d39580ea4ff970deae32b9f Mon Sep 17 00:00:00 2001 From: bastimeyer Date: Mon, 29 Jan 2024 20:13:15 +0100 Subject: [PATCH 2/3] session: re-use PathEntryFinder in load_plugins() --- src/streamlink/session.py | 7 ++++--- tests/test_session.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/streamlink/session.py b/src/streamlink/session.py index a120d794318..53c2fc726e8 100644 --- a/src/streamlink/session.py +++ b/src/streamlink/session.py @@ -15,7 +15,7 @@ from streamlink.plugin.api.http_session import HTTPSession, TLSNoDHAdapter from streamlink.plugin.plugin import NO_PRIORITY, Matcher, Plugin from streamlink.utils.l10n import Localization -from streamlink.utils.module import load_module +from streamlink.utils.module import exec_module from streamlink.utils.url import update_scheme @@ -641,12 +641,13 @@ def load_plugins(self, path: str) -> bool: """ success = False - for _loader, name, _ispkg in pkgutil.iter_modules([path]): + for module_info in pkgutil.iter_modules([path]): + name = module_info.name # set the full plugin module name # use the "streamlink.plugins." prefix even for sideloaded plugins module_name = f"streamlink.plugins.{name}" try: - mod = load_module(module_name, path) + mod = exec_module(module_info.module_finder, module_name) # type: ignore[arg-type] except ImportError as err: log.exception(f"Failed to load plugin {name} from {path}", exc_info=err) continue diff --git a/tests/test_session.py b/tests/test_session.py index fa2bdd28e19..39bf5b8e265 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -103,7 +103,7 @@ def test_load_plugins_failure( logs: list, ): monkeypatch.setattr("streamlink.session.Streamlink.load_builtin_plugins", Mock()) - monkeypatch.setattr("streamlink.session.load_module", Mock(side_effect=side_effect)) + monkeypatch.setattr("streamlink.session.exec_module", Mock(side_effect=side_effect)) session = Streamlink() with raises: session.load_plugins(str(PATH_TESTPLUGINS)) From 43e0b5372f543657de685c8bd53bf185926e1bc7 Mon Sep 17 00:00:00 2001 From: bastimeyer Date: Mon, 29 Jan 2024 20:15:11 +0100 Subject: [PATCH 3/3] tests: re-use PathEntryFinder in test_plugins --- tests/test_plugins.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index d2271675011..69dc414cf44 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,7 +9,7 @@ import streamlink.plugins import tests.plugins from streamlink.plugin.plugin import Matcher, Plugin -from streamlink.utils.module import load_module +from streamlink.utils.module import exec_module plugins_path = streamlink.plugins.__path__[0] @@ -24,11 +24,12 @@ "test_stream", ] -plugins = [ - pname - for finder, pname, ispkg in pkgutil.iter_modules([plugins_path]) - if not pname.startswith("common_") +plugin_modules = [ + module_info + for module_info in pkgutil.iter_modules([plugins_path]) + if not module_info.name.startswith("common_") ] +plugins = [module_info.name for module_info in plugin_modules] plugins_no_protocols = [pname for pname in plugins if pname not in protocol_plugins] plugintests = [ re.sub(r"^test_", "", tname) @@ -52,9 +53,9 @@ def unique(iterable): class TestPlugins: - @pytest.fixture(scope="class", params=plugins) + @pytest.fixture(scope="class", params=plugin_modules) def plugin(self, request): - return load_module(f"streamlink.plugins.{request.param}", plugins_path) + return exec_module(request.param.module_finder, f"streamlink.plugins.{request.param.name}") def test_exports_plugin(self, plugin): assert hasattr(plugin, "__plugin__"), "Plugin module exports __plugin__"