From 969f2372231e62dc201207c9301a37a8f07be41f Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 10 Feb 2021 17:26:30 +0100 Subject: [PATCH 1/6] Refactor ownership in .NET Framework loader --- netfx_loader/ClrLoader.cs | 74 ++++++++++++++++++++++---------------- netfx_loader/DomainData.cs | 61 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 30 deletions(-) create mode 100644 netfx_loader/DomainData.cs diff --git a/netfx_loader/ClrLoader.cs b/netfx_loader/ClrLoader.cs index 2ba4e74..e8b2767 100644 --- a/netfx_loader/ClrLoader.cs +++ b/netfx_loader/ClrLoader.cs @@ -1,8 +1,6 @@ using System; -using System.Diagnostics; -using System.Globalization; +using System.Collections.Generic; using System.IO; -using System.Reflection; using System.Runtime.InteropServices; using NXPorts.Attributes; @@ -10,7 +8,18 @@ namespace ClrLoader { public static class ClrLoader { - delegate int EntryPoint(IntPtr buffer, int size); + static bool _initialized = false; + static List _domains = new List(); + + [DllExport("pyclr_initialize", CallingConvention.Cdecl)] + public static void Initialize() + { + if (!_initialized) + { + _domains.Add(new DomainData(AppDomain.CurrentDomain)); + _initialized = true; + } + } [DllExport("pyclr_create_appdomain", CallingConvention.Cdecl)] public static IntPtr CreateAppDomain( @@ -29,11 +38,9 @@ public static IntPtr CreateAppDomain( Print($"Located domain {domain}"); - var handle = GCHandle.Alloc(domain, GCHandleType.Pinned); - - Print($"Created handle {handle}"); - - return handle.AddrOfPinnedObject(); + var domainData = new DomainData(domain); + _domains.Add(domainData); + return new IntPtr(_domains.Count - 1); } else { @@ -51,18 +58,8 @@ public static IntPtr GetFunction( { try { - var domainObj = AppDomain.CurrentDomain; - if (domain != IntPtr.Zero) - { - var handle = GCHandle.FromIntPtr(domain); - domainObj = (AppDomain)handle.Target; - } - - var assembly = domainObj.Load(AssemblyName.GetAssemblyName(assemblyPath)); - var type = assembly.GetType(typeName, throwOnError: true); - Print($"Loaded type {type}"); - var deleg = Delegate.CreateDelegate(typeof(EntryPoint), type, function); - + var domainData = _domains[(int)domain]; + var deleg = domainData.GetEntryPoint(assemblyPath, typeName, function); return Marshal.GetFunctionPointerForDelegate(deleg); } catch (Exception exc) @@ -77,21 +74,38 @@ public static void CloseAppDomain(IntPtr domain) { if (domain != IntPtr.Zero) { - var handle = GCHandle.FromIntPtr(domain); - var domainObj = (AppDomain)handle.Target; - AppDomain.Unload(domainObj); - handle.Free(); + try + { + var domainData = _domains[(int)domain]; + domainData.Dispose(); + } + catch (Exception exc) + { + Print($"Exception in {nameof(CloseAppDomain)}: {exc.GetType().Name} {exc.Message}\n{exc.StackTrace}"); + } + } + } + + [DllExport("pyclr_finalize", CallingConvention.Cdecl)] + public static void Close() + { + foreach (var domainData in _domains) + { + domainData.Dispose(); } + + _domains.Clear(); + _initialized = false; } - #if DEBUG - static void Print(string s) +#if DEBUG + internal static void Print(string s) { Console.WriteLine(s); } - #else - static void Print(string s) {} - #endif +#else + internal static void Print(string s) { } +#endif } } diff --git a/netfx_loader/DomainData.cs b/netfx_loader/DomainData.cs new file mode 100644 index 0000000..3a17d7a --- /dev/null +++ b/netfx_loader/DomainData.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace ClrLoader +{ + using static ClrLoader; + + class DomainData : IDisposable + { + public delegate int EntryPoint(IntPtr buffer, int size); + + bool _disposed = false; + + public AppDomain Domain { get; } + public Dictionary<(string, string, string), EntryPoint> _delegates; + + public DomainData(AppDomain domain) + { + Domain = domain; + _delegates = new Dictionary<(string, string, string), EntryPoint>(); + } + + public EntryPoint GetEntryPoint(string assemblyPath, string typeName, string function) + { + if (_disposed) + throw new InvalidOperationException("Domain is already disposed"); + + var key = (assemblyPath, typeName, function); + + EntryPoint result; + + if (!_delegates.TryGetValue(key, out result)) + { + var assembly = Domain.Load(AssemblyName.GetAssemblyName(assemblyPath)); + var type = assembly.GetType(typeName, throwOnError: true); + + Print($"Loaded type {type}"); + result = (EntryPoint)Delegate.CreateDelegate(typeof(EntryPoint), type, function); + + _delegates[key] = result; + } + + return result; + } + + public void Dispose() + { + if (!_disposed) + { + _delegates.Clear(); + + if (Domain != AppDomain.CurrentDomain) + AppDomain.Unload(Domain); + + _disposed = true; + } + } + + } +} \ No newline at end of file From 367d70823998a9d1e697569dae1962012c456984 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 10 Feb 2021 17:31:25 +0100 Subject: [PATCH 2/6] Call initialize and finalize from Python --- clr_loader/ffi/netfx.py | 2 ++ clr_loader/netfx.py | 25 ++++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/clr_loader/ffi/netfx.py b/clr_loader/ffi/netfx.py index 02a7737..2ebc862 100644 --- a/clr_loader/ffi/netfx.py +++ b/clr_loader/ffi/netfx.py @@ -5,8 +5,10 @@ typedef void* pyclr_domain; typedef int (*entry_point)(void* buffer, int size); +void pyclr_initialize(); void* pyclr_create_appdomain(const char* name, const char* config_file); entry_point pyclr_get_function(pyclr_domain domain, const char* assembly_path, const char* class_name, const char* function); void pyclr_close_appdomain(pyclr_domain domain); +void pyclr_finalize(); """ ] diff --git a/clr_loader/netfx.py b/clr_loader/netfx.py index 5c6e794..4ca0d62 100644 --- a/clr_loader/netfx.py +++ b/clr_loader/netfx.py @@ -1,15 +1,12 @@ +import atexit from .ffi import ffi, load_netfx - _FW = None class NetFx: def __init__(self, name=None, config_file=None): - global _FW - if _FW is None: - _FW = load_netfx() - + initialize() self._domain = _FW.pyclr_create_appdomain( name or ffi.NULL, config_file or ffi.NULL ) @@ -27,3 +24,21 @@ def get_callable(self, assembly_path, typename, function): def __del__(self): if self._domain and _FW: _FW.pyclr_close_appdomain(self._domain) + + +def initialize(): + global _FW + if _FW is not None: + return + + _FW = load_netfx() + _FW.pyclr_initialize() + + atexit.register(_release) + + +def _release(): + global _FW + if _FW is not None: + _FW.pyclr_finalize() + _FW = None From 2287199b25a1594e4e773ee0151abd0c9ac9300e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 18:15:34 +0100 Subject: [PATCH 3/6] Fix passing properties to .NET Core init and add type hints --- clr_loader/__init__.py | 2 +- clr_loader/hostfxr.py | 10 +++++----- clr_loader/mono.py | 6 +++--- clr_loader/netfx.py | 7 ++++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/clr_loader/__init__.py b/clr_loader/__init__.py index 26dcee0..e8de6be 100644 --- a/clr_loader/__init__.py +++ b/clr_loader/__init__.py @@ -33,7 +33,7 @@ def get_coreclr( impl = DotnetCoreRuntime(runtime_config=runtime_config, dotnet_root=dotnet_root) if properties: - for key, value in properties: + for key, value in properties.items(): impl[key] = value return Runtime(impl) diff --git a/clr_loader/hostfxr.py b/clr_loader/hostfxr.py index f7a437c..63a7439 100644 --- a/clr_loader/hostfxr.py +++ b/clr_loader/hostfxr.py @@ -16,11 +16,11 @@ def __init__(self, runtime_config: str, dotnet_root: str): self._load_func = _get_load_func(self._dll, self._handle) @property - def dotnet_root(self): + def dotnet_root(self) -> str: return self._dotnet_root @property - def is_finalized(self): + def is_finalized(self) -> bool: return self._is_finalized def __getitem__(self, key: str) -> str: @@ -58,7 +58,7 @@ def __iter__(self): for i in range(size_ptr[0]): yield (decode(keys_ptr[i]), decode(values_ptr[i])) - def get_callable(self, assembly_path, typename, function): + def get_callable(self, assembly_path: str, typename: str, function: str): # TODO: Maybe use coreclr_get_delegate as well, supported with newer API # versions of hostfxr self._is_finalized = True @@ -79,7 +79,7 @@ def get_callable(self, assembly_path, typename, function): check_result(res) return ffi.cast("component_entry_point_fn", delegate_ptr[0]) - def shutdown(self): + def shutdown(self) -> None: if self._handle is not None: self._dll.hostfxr_close(self._handle) self._handle = None @@ -88,7 +88,7 @@ def __del__(self): self.shutdown() -def _get_handle(dll, dotnet_root, runtime_config): +def _get_handle(dll, dotnet_root: str, runtime_config: str): params = ffi.new("hostfxr_initialize_parameters*") params.size = ffi.sizeof("hostfxr_initialize_parameters") # params.host_path = ffi.new("char_t[]", encode(sys.executable)) diff --git a/clr_loader/mono.py b/clr_loader/mono.py index 771d4d6..8af4dd3 100644 --- a/clr_loader/mono.py +++ b/clr_loader/mono.py @@ -87,12 +87,12 @@ def initialize(config_file: str, libmono: str) -> None: _MONO = load_mono(libmono) if config_file is None: - config_file = ffi.NULL + config_bytes = ffi.NULL else: - config_file = config_file.encode("utf8") + config_bytes = config_file.encode("utf8") _ROOT_DOMAIN = _MONO.mono_jit_init(b"clr_loader") - _MONO.mono_config_parse(config_file) + _MONO.mono_config_parse(config_bytes) _check_result(_ROOT_DOMAIN, "Failed to initialize Mono") atexit.register(_release) diff --git a/clr_loader/netfx.py b/clr_loader/netfx.py index 4ca0d62..6b6c003 100644 --- a/clr_loader/netfx.py +++ b/clr_loader/netfx.py @@ -1,17 +1,18 @@ import atexit +from typing import Optional, Any from .ffi import ffi, load_netfx -_FW = None +_FW: Optional[Any] = None class NetFx: - def __init__(self, name=None, config_file=None): + def __init__(self, name: Optional[str] = None, config_file: Optional[str] = None): initialize() self._domain = _FW.pyclr_create_appdomain( name or ffi.NULL, config_file or ffi.NULL ) - def get_callable(self, assembly_path, typename, function): + def get_callable(self, assembly_path: str, typename: str, function: str): func = _FW.pyclr_get_function( self._domain, assembly_path.encode("utf8"), From a858d896b189c0cd4599ca61bb36be5b828f6641 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 18:24:54 +0100 Subject: [PATCH 4/6] Fix runtime property iteration --- clr_loader/hostfxr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clr_loader/hostfxr.py b/clr_loader/hostfxr.py index 63a7439..fed2f77 100644 --- a/clr_loader/hostfxr.py +++ b/clr_loader/hostfxr.py @@ -50,8 +50,8 @@ def __iter__(self): keys_ptr = ffi.new("char_t*[]", max_size) values_ptr = ffi.new("char_t*[]", max_size) - res = self._fxr._dll.hostfxr_get_runtime_properties( - self._fxr._handle, size_ptr, keys_ptr, values_ptr + res = self._dll.hostfxr_get_runtime_properties( + self._dll._handle, size_ptr, keys_ptr, values_ptr ) check_result(res) From a0128863bfde444c552fdf82aa789a15bca7b0b1 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 18:37:25 +0100 Subject: [PATCH 5/6] Fix small lints and drop obsolete parameters --- clr_loader/ffi/__init__.py | 2 +- clr_loader/mono.py | 4 ++-- clr_loader/util/clr_error.py | 1 + clr_loader/util/find.py | 2 +- clr_loader/wrappers.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clr_loader/ffi/__init__.py b/clr_loader/ffi/__init__.py index ef46a84..114803c 100644 --- a/clr_loader/ffi/__init__.py +++ b/clr_loader/ffi/__init__.py @@ -28,7 +28,7 @@ def load_hostfxr(dotnet_root: str): raise RuntimeError(f"Could not find a suitable hostfxr library in {dotnet_root}") -def load_mono(path: Optional[str] = None, gc: Optional[str] = None): +def load_mono(path: Optional[str] = None): # Preload C++ standard library, Mono needs that and doesn't properly link against it if sys.platform.startswith("linux"): ffi.dlopen("stdc++", ffi.RTLD_GLOBAL) diff --git a/clr_loader/mono.py b/clr_loader/mono.py index 8af4dd3..d3991f7 100644 --- a/clr_loader/mono.py +++ b/clr_loader/mono.py @@ -39,7 +39,7 @@ def get_callable(self, assembly_path, typename, function): method, f"Could not find method {typename}.{function} in assembly" ) - return MonoMethod(self._domain, method) + return MonoMethod(method) class MethodDesc: @@ -58,7 +58,7 @@ def __del__(self): class MonoMethod: - def __init__(self, domain, ptr): + def __init__(self, ptr): self._ptr = ptr def __call__(self, ptr, size): diff --git a/clr_loader/util/clr_error.py b/clr_loader/util/clr_error.py index eef4d2c..91c0c10 100644 --- a/clr_loader/util/clr_error.py +++ b/clr_loader/util/clr_error.py @@ -13,6 +13,7 @@ def __init__( self.name = name self.message = message self.comment = comment + super().__init__(self.message) def __str__(self): if self.message: diff --git a/clr_loader/util/find.py b/clr_loader/util/find.py index 578e672..f33b0c2 100644 --- a/clr_loader/util/find.py +++ b/clr_loader/util/find.py @@ -69,5 +69,5 @@ def find_libmono(sgen: bool = True) -> str: if path is None: raise RuntimeError("Could not find libmono") - + return path diff --git a/clr_loader/wrappers.py b/clr_loader/wrappers.py index 805fcbf..e8f2a89 100644 --- a/clr_loader/wrappers.py +++ b/clr_loader/wrappers.py @@ -1,6 +1,6 @@ +from os.path import basename from typing import Any, Optional from .ffi import ffi -from os.path import basename RuntimeImpl = Any From e90553bfc74de93ea4a1803d4791dc6512b1cbe0 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 16:46:10 +0100 Subject: [PATCH 6/6] Bump version to 0.1.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a867f9..27083cd 100755 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def finalize_options(self): setup( name="clr_loader", - version="0.1.4", + version="0.1.5", description="Generic pure Python loader for .NET runtimes", author="Benedikt Reinartz", author_email="filmor@gmail.com",