From 79d6c6792ebc09e43d514a3ce554b32a89a2eb19 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Fri, 13 Jan 2023 04:52:54 -0600 Subject: [PATCH 1/8] fix mono set dirs (#48) * fix mono set dirs * fix path --- clr_loader/__init__.py | 2 +- clr_loader/mono.py | 2 +- clr_loader/util/find.py | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/clr_loader/__init__.py b/clr_loader/__init__.py index 4b08148..aa604ad 100644 --- a/clr_loader/__init__.py +++ b/clr_loader/__init__.py @@ -70,7 +70,7 @@ def get_mono( libmono = _maybe_path(libmono) if libmono is None: - libmono = find_libmono(sgen=sgen) + libmono = find_libmono(sgen=sgen, assembly_dir=assembly_dir) impl = Mono( # domain=domain, diff --git a/clr_loader/mono.py b/clr_loader/mono.py index 158ddb7..7c3f20d 100644 --- a/clr_loader/mono.py +++ b/clr_loader/mono.py @@ -136,7 +136,7 @@ def initialize( _MONO = load_mono(libmono) if assembly_dir is not None and config_dir is not None: - _MONO.mono_set_dirs(assembly_dir, config_dir) + _MONO.mono_set_dirs(assembly_dir.encode("utf8"), config_dir.encode("utf8")) # Load in global config (i.e /etc/mono/config) global_encoded = global_config_file or ffi.NULL diff --git a/clr_loader/util/find.py b/clr_loader/util/find.py index d5d7b89..6ef7bc3 100644 --- a/clr_loader/util/find.py +++ b/clr_loader/util/find.py @@ -105,7 +105,7 @@ def find_runtimes() -> Iterator[DotnetCoreRuntimeSpec]: return find_runtimes_in_root(dotnet_root) -def find_libmono(*, sgen: bool = True) -> Path: +def find_libmono(*, assembly_dir: str = None, sgen: bool = True) -> Path: """Find a suitable libmono dynamic library On Windows and macOS, we check the default installation directories. @@ -137,9 +137,12 @@ def find_libmono(*, sgen: bool = True) -> Path: ) else: - from ctypes.util import find_library - - path = find_library(unix_name) + if assembly_dir == None: + from ctypes.util import find_library + path = find_library(unix_name) + else: + libname = "lib" + unix_name + ".so" + path = Path(assembly_dir) / "lib" / libname if path is None: raise RuntimeError("Could not find libmono") From 7dd9b463099425087f4de51c7a68b365a4612344 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 7 Aug 2023 14:10:46 +0200 Subject: [PATCH 2/8] Add Python 3.11 to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6f9e75..0c6749e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python: ['3.10', '3.9', '3.8', '3.7'] # pypy3 + python: ['3.11', '3.10', '3.9', '3.8', '3.7'] # pypy3 steps: - uses: actions/checkout@v3 From d8f51e9e28f41f2a495327e7cbecb50edd6af243 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 7 Aug 2023 14:33:45 +0200 Subject: [PATCH 3/8] Drop CI for EOLd Python 3.7 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c6749e..41f4e0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python: ['3.11', '3.10', '3.9', '3.8', '3.7'] # pypy3 + python: ['3.11', '3.10', '3.9', '3.8'] # pypy3 steps: - uses: actions/checkout@v3 From 63d7f698d3be00bd1f609545e12f87cc3c549c42 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 7 Aug 2023 14:39:44 +0200 Subject: [PATCH 4/8] Fix handling of non-ASCII assembly paths on .NET Framework (#62) --- netfx_loader/ClrLoader.cs | 6 +++--- tests/test_common.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/netfx_loader/ClrLoader.cs b/netfx_loader/ClrLoader.cs index e8b2767..af89cee 100644 --- a/netfx_loader/ClrLoader.cs +++ b/netfx_loader/ClrLoader.cs @@ -51,9 +51,9 @@ public static IntPtr CreateAppDomain( [DllExport("pyclr_get_function", CallingConvention.Cdecl)] public static IntPtr GetFunction( IntPtr domain, - [MarshalAs(UnmanagedType.LPStr)] string assemblyPath, - [MarshalAs(UnmanagedType.LPStr)] string typeName, - [MarshalAs(UnmanagedType.LPStr)] string function + [MarshalAs(UnmanagedType.LPUTF8Str)] string assemblyPath, + [MarshalAs(UnmanagedType.LPUTF8Str)] string typeName, + [MarshalAs(UnmanagedType.LPUTF8Str)] string function ) { try diff --git a/tests/test_common.py b/tests/test_common.py index f42020a..250c290 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,3 +1,4 @@ +import shutil import pytest from subprocess import check_call import os @@ -24,7 +25,7 @@ def build_example(tmpdir_factory, framework): return out -def test_mono(example_netstandard): +def test_mono(example_netstandard: Path): from clr_loader import get_mono mono = get_mono() @@ -33,7 +34,7 @@ def test_mono(example_netstandard): run_tests(asm) -def test_mono_debug(example_netstandard): +def test_mono_debug(example_netstandard: Path): from clr_loader import get_mono mono = get_mono( @@ -46,7 +47,8 @@ def test_mono_debug(example_netstandard): run_tests(asm) -def test_mono_signal_chaining(example_netstandard): + +def test_mono_signal_chaining(example_netstandard: Path): from clr_loader import get_mono mono = get_mono(set_signal_chaining=True) @@ -54,7 +56,8 @@ def test_mono_signal_chaining(example_netstandard): run_tests(asm) -def test_mono_set_dir(example_netstandard): + +def test_mono_set_dir(example_netstandard: Path): from clr_loader import get_mono mono = get_mono(assembly_dir="/usr/lib", config_dir="/etc") @@ -62,7 +65,8 @@ def test_mono_set_dir(example_netstandard): run_tests(asm) -def test_coreclr(example_netcore): + +def test_coreclr(example_netcore: Path): from clr_loader import get_coreclr coreclr = get_coreclr(runtime_config=example_netcore / "example.runtimeconfig.json") @@ -71,7 +75,7 @@ def test_coreclr(example_netcore): run_tests(asm) -def test_coreclr_autogenerated_runtimeconfig(example_netstandard): +def test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): from multiprocessing import get_context p = get_context("spawn").Process( @@ -82,7 +86,7 @@ def test_coreclr_autogenerated_runtimeconfig(example_netstandard): p.close() -def _do_test_coreclr_autogenerated_runtimeconfig(example_netstandard): +def _do_test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): from clr_loader import get_coreclr coreclr = get_coreclr() @@ -94,9 +98,24 @@ def _do_test_coreclr_autogenerated_runtimeconfig(example_netstandard): @pytest.mark.skipif( sys.platform != "win32", reason=".NET Framework only exists on Windows" ) -def test_netfx(example_netstandard): +def test_netfx(example_netstandard: Path): + from clr_loader import get_netfx + + netfx = get_netfx() + asm = netfx.get_assembly(example_netstandard / "example.dll") + + run_tests(asm) + + +@pytest.mark.skipif( + sys.platform != "win32", reason=".NET Framework only exists on Windows" +) +def test_netfx_chinese_path(example_netstandard: Path, tmpdir_factory): from clr_loader import get_netfx + tmp_path = Path(tmpdir_factory.mktemp("example-中国")) + shutil.copytree(example_netstandard, tmp_path, dirs_exist_ok=True) + netfx = get_netfx() asm = netfx.get_assembly(os.path.join(example_netstandard, "example.dll")) From 000bc780b33b5dbb3f0c1891776fda0971cc2df2 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 7 Aug 2023 15:21:26 +0200 Subject: [PATCH 5/8] Delay actually loading the runtime to fix property setting (#63) * Delay actually loading the runtime to fix property setting * Launch test for properties in separate process --- clr_loader/hostfxr.py | 20 +++++++++----------- tests/test_common.py | 21 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/clr_loader/hostfxr.py b/clr_loader/hostfxr.py index 522fec5..225b4c7 100644 --- a/clr_loader/hostfxr.py +++ b/clr_loader/hostfxr.py @@ -20,9 +20,8 @@ def __init__(self, runtime_config: Path, dotnet_root: Path, **params: str): self._dotnet_root = Path(dotnet_root) self._dll = load_hostfxr(self._dotnet_root) - self._is_initialized = False self._handle = _get_handle(self._dll, self._dotnet_root, runtime_config) - self._load_func = _get_load_func(self._dll, self._handle) + self._load_func = None for key, value in params.items(): self[key] = value @@ -36,7 +35,7 @@ def dotnet_root(self) -> Path: @property def is_initialized(self) -> bool: - return self._is_initialized + return self._load_func is not None @property def is_shutdown(self) -> bool: @@ -81,10 +80,15 @@ def __iter__(self) -> Generator[Tuple[str, str], None, None]: for i in range(size_ptr[0]): yield (decode(keys_ptr[i]), decode(values_ptr[i])) + def _get_load_func(self): + if self._load_func is None: + self._load_func = _get_load_func(self._dll, self._handle) + + return self._load_func + def _get_callable(self, assembly_path: StrOrPath, typename: str, function: str): # TODO: Maybe use coreclr_get_delegate as well, supported with newer API # versions of hostfxr - self._is_initialized = True # Append assembly name to typename assembly_path = Path(assembly_path) @@ -92,7 +96,7 @@ def _get_callable(self, assembly_path: StrOrPath, typename: str, function: str): typename = f"{typename}, {assembly_name}" delegate_ptr = ffi.new("void**") - res = self._load_func( + res = self._get_load_func()( encode(str(assembly_path)), encode(typename), encode(function), @@ -103,12 +107,6 @@ def _get_callable(self, assembly_path: StrOrPath, typename: str, function: str): check_result(res) return ffi.cast("component_entry_point_fn", delegate_ptr[0]) - def _check_initialized(self) -> None: - if self._handle is None: - raise RuntimeError("Runtime is shut down") - elif not self._is_initialized: - raise RuntimeError("Runtime is not initialized") - def shutdown(self) -> None: if self._handle is not None: self._dll.hostfxr_close(self._handle) diff --git a/tests/test_common.py b/tests/test_common.py index 250c290..a33dec0 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -18,7 +18,7 @@ def example_netcore(tmpdir_factory): def build_example(tmpdir_factory, framework): out = Path(tmpdir_factory.mktemp(f"example-{framework}")) - proj_path = Path(__file__).parent.parent / "example" + proj_path = Path(__file__).parent.parent / "example" / "example.csproj" check_call(["dotnet", "build", str(proj_path), "-o", str(out), "-f", framework]) @@ -75,6 +75,19 @@ def test_coreclr(example_netcore: Path): run_tests(asm) +def test_coreclr_properties(example_netcore: Path): + from multiprocessing import get_context + + p = get_context("spawn").Process( + target=_do_test_coreclr_autogenerated_runtimeconfig, + args=(example_netstandard,), + kwargs=dict(properties=dict(APP_CONTEXT_BASE_DIRECTORY=str(example_netcore))), + ) + p.start() + p.join() + p.close() + + def test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): from multiprocessing import get_context @@ -86,10 +99,12 @@ def test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): p.close() -def _do_test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): +def _do_test_coreclr_autogenerated_runtimeconfig( + example_netstandard: Path, **properties +): from clr_loader import get_coreclr - coreclr = get_coreclr() + coreclr = get_coreclr(properties=properties) asm = coreclr.get_assembly(example_netstandard / "example.dll") run_tests(asm) From 70e2c72da9db0c769df1df195891da5c74a55dd5 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 16 Dec 2022 16:33:59 +0100 Subject: [PATCH 6/8] Fix parameter passing for .NET Framework domains --- clr_loader/netfx.py | 11 +++++++++-- netfx_loader/ClrLoader.cs | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/clr_loader/netfx.py b/clr_loader/netfx.py index 24460ca..4efe841 100644 --- a/clr_loader/netfx.py +++ b/clr_loader/netfx.py @@ -14,13 +14,15 @@ def __init__( ): initialize() if config_file is not None: - config_file_s = str(config_file) + config_file_s = str(config_file).encode("utf8") else: config_file_s = ffi.NULL + domain_s = domain.encode("utf8") if domain else ffi.NULL + self._domain_name = domain self._config_file = config_file - self._domain = _FW.pyclr_create_appdomain(domain or ffi.NULL, config_file_s) + self._domain = _FW.pyclr_create_appdomain(domain_s, config_file_s) def info(self) -> RuntimeInfo: return RuntimeInfo( @@ -41,6 +43,11 @@ def _get_callable(self, assembly_path: StrOrPath, typename: str, function: str): function.encode("utf8"), ) + if func == ffi.NULL: + raise RuntimeError( + f"Failed to resolve {typename}.{function} from {assembly_path}" + ) + return func def shutdown(self): diff --git a/netfx_loader/ClrLoader.cs b/netfx_loader/ClrLoader.cs index af89cee..32b4c01 100644 --- a/netfx_loader/ClrLoader.cs +++ b/netfx_loader/ClrLoader.cs @@ -32,8 +32,10 @@ public static IntPtr CreateAppDomain( { var setup = new AppDomainSetup { + ApplicationBase = AppDomain.CurrentDomain.BaseDirectory, ConfigurationFile = configFile }; + Print($"Base: {AppDomain.CurrentDomain.BaseDirectory}"); var domain = AppDomain.CreateDomain(name, null, setup); Print($"Located domain {domain}"); From 05235b979a6fa169898318af7d752428a1bbd850 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 16 Dec 2022 16:40:08 +0100 Subject: [PATCH 7/8] Add unit test for using a separate domain --- tests/test_common.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_common.py b/tests/test_common.py index a33dec0..3cba3cd 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -137,6 +137,18 @@ def test_netfx_chinese_path(example_netstandard: Path, tmpdir_factory): run_tests(asm) +@pytest.mark.skipif( + sys.platform != "win32", reason=".NET Framework only exists on Windows" +) +def test_netfx_separate_domain(example_netstandard): + from clr_loader import get_netfx + + netfx = get_netfx(domain="some_domain") + asm = netfx.get_assembly(os.path.join(example_netstandard, "example.dll")) + + run_tests(asm) + + def run_tests(asm): func = asm.get_function("Example.TestClass", "Test") test_data = b"testy mctestface" From 59f47f9088da9cf999439bb15ff7ac131633fec7 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 7 Aug 2023 16:22:10 +0200 Subject: [PATCH 8/8] Move netfx tests to individual subprocesses --- tests/test_common.py | 53 ++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 3cba3cd..8a9e36d 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -76,27 +76,15 @@ def test_coreclr(example_netcore: Path): def test_coreclr_properties(example_netcore: Path): - from multiprocessing import get_context - - p = get_context("spawn").Process( - target=_do_test_coreclr_autogenerated_runtimeconfig, - args=(example_netstandard,), - kwargs=dict(properties=dict(APP_CONTEXT_BASE_DIRECTORY=str(example_netcore))), + run_in_subprocess( + _do_test_coreclr_autogenerated_runtimeconfig, + example_netstandard, + properties=dict(APP_CONTEXT_BASE_DIRECTORY=str(example_netcore)), ) - p.start() - p.join() - p.close() def test_coreclr_autogenerated_runtimeconfig(example_netstandard: Path): - from multiprocessing import get_context - - p = get_context("spawn").Process( - target=_do_test_coreclr_autogenerated_runtimeconfig, args=(example_netstandard,) - ) - p.start() - p.join() - p.close() + run_in_subprocess(_do_test_coreclr_autogenerated_runtimeconfig, example_netstandard) def _do_test_coreclr_autogenerated_runtimeconfig( @@ -114,37 +102,31 @@ def _do_test_coreclr_autogenerated_runtimeconfig( sys.platform != "win32", reason=".NET Framework only exists on Windows" ) def test_netfx(example_netstandard: Path): - from clr_loader import get_netfx - - netfx = get_netfx() - asm = netfx.get_assembly(example_netstandard / "example.dll") - - run_tests(asm) + run_in_subprocess(_do_test_netfx, example_netstandard) @pytest.mark.skipif( sys.platform != "win32", reason=".NET Framework only exists on Windows" ) def test_netfx_chinese_path(example_netstandard: Path, tmpdir_factory): - from clr_loader import get_netfx - tmp_path = Path(tmpdir_factory.mktemp("example-中国")) shutil.copytree(example_netstandard, tmp_path, dirs_exist_ok=True) - netfx = get_netfx() - asm = netfx.get_assembly(os.path.join(example_netstandard, "example.dll")) - - run_tests(asm) + run_in_subprocess(_do_test_netfx, tmp_path) @pytest.mark.skipif( sys.platform != "win32", reason=".NET Framework only exists on Windows" ) def test_netfx_separate_domain(example_netstandard): + run_in_subprocess(_do_test_netfx, example_netstandard, domain="some domain") + + +def _do_test_netfx(example_netstandard, **kwargs): from clr_loader import get_netfx - netfx = get_netfx(domain="some_domain") - asm = netfx.get_assembly(os.path.join(example_netstandard, "example.dll")) + netfx = get_netfx(**kwargs) + asm = netfx.get_assembly(example_netstandard / "example.dll") run_tests(asm) @@ -154,3 +136,12 @@ def run_tests(asm): test_data = b"testy mctestface" res = func(test_data) assert res == len(test_data) + + +def run_in_subprocess(func, *args, **kwargs): + from multiprocessing import get_context + + p = get_context("spawn").Process(target=func, args=args, kwargs=kwargs) + p.start() + p.join() + p.close()