From 1dd2ee1b02449b85eeee6120c88a4092dc78851a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 25 Jun 2019 23:00:34 +0200 Subject: [PATCH 1/2] Get the correct library loading functions at runtime --- src/embed_tests/TestRuntime.cs | 7 +- src/runtime/Python.Runtime.csproj | 2 + src/runtime/platform/LibraryLoader.cs | 153 ++++++++++++++++++++++++++ src/runtime/platform/Types.cs | 22 ++++ src/runtime/runtime.cs | 128 ++------------------- src/runtime/typemanager.cs | 22 ++-- 6 files changed, 203 insertions(+), 131 deletions(-) create mode 100644 src/runtime/platform/LibraryLoader.cs create mode 100644 src/runtime/platform/Types.cs diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..25b70fac5 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,6 +1,7 @@ using System; using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; namespace Python.EmbeddingTest { @@ -26,10 +27,10 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); + Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); + Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); // Don't shut down the runtime: if the python engine was initialized @@ -39,7 +40,7 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 19f776c77..c4d63695f 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -140,6 +140,8 @@ + + diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs new file mode 100644 index 000000000..c0157b04b --- /dev/null +++ b/src/runtime/platform/LibraryLoader.cs @@ -0,0 +1,153 @@ +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibraryLoader + { + IntPtr Load(string dllToLoad); + + IntPtr GetFunction(IntPtr hModule, string procedureName); + + bool Free(IntPtr hModule); + } + + static class LibraryLoader + { + public static ILibraryLoader Get(OperatingSystemType os) + { + switch (os) + { + case OperatingSystemType.Windows: + return new WindowsLoader(); + case OperatingSystemType.Darwin: + return new DarwinLoader(); + case OperatingSystemType.Linux: + return new LinuxLoader(); + default: + throw new Exception($"This operating system ({os}) is not supported"); + } + } + } + + class LinuxLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x100; + private static IntPtr RTLD_DEFAULT = IntPtr.Zero; + private const string NativeDll = "libdl.so"; + + public IntPtr Load(string fileName) + { + return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); + } + + public bool Free(IntPtr handle) + { + dlclose(handle); + return true; + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + // clear previous errors if any + dlerror(); + IntPtr res = dlsym(dllHandle, name); + IntPtr errPtr = dlerror(); + if (errPtr != IntPtr.Zero) + { + throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + } + return res; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class DarwinLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x8; + private const string NativeDll = "/usr/lib/libSystem.dylib"; + private static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + public IntPtr Load(string fileName) + { + return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); + } + + public bool Free(IntPtr handle) + { + dlclose(handle); + return true; + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + // clear previous errors if any + dlerror(); + IntPtr res = dlsym(dllHandle, name); + IntPtr errPtr = dlerror(); + if (errPtr != IntPtr.Zero) + { + throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + } + return res; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class WindowsLoader : ILibraryLoader + { + private const string NativeDll = "kernel32.dll"; + + [DllImport(NativeDll)] + static extern IntPtr LoadLibrary(string dllToLoad); + + public IntPtr Load(string dllToLoad) => WindowsLoader.LoadLibrary(dllToLoad); + + [DllImport(NativeDll)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + public IntPtr GetFunction(IntPtr hModule, string procedureName) => WindowsLoader.GetProcAddress(hModule, procedureName); + + + [DllImport(NativeDll)] + static extern bool FreeLibrary(IntPtr hModule); + + public bool Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs new file mode 100644 index 000000000..bdc51af39 --- /dev/null +++ b/src/runtime/platform/Types.cs @@ -0,0 +1,22 @@ +namespace Python.Runtime.Platform +{ + public enum MachineType + { + i386, + x86_64, + armv7l, + armv8, + Other + }; + + /// + /// Operating system type as reported by Python. + /// + public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 294ecaf48..ec5bddfd0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,96 +7,7 @@ namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } + using Python.Runtime.Platform; /// /// Encapsulates the low-level Python C API. Note that it is @@ -197,17 +108,6 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() { { "Windows", OperatingSystemType.Windows }, @@ -225,14 +125,6 @@ public enum OperatingSystemType /// public static string OperatingSystemName { get; private set; } - public enum MachineType - { - i386, - x86_64, - armv7l, - armv8, - Other - }; /// /// Map lower-case version of the python machine name to the processor @@ -397,24 +289,24 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + InitializePlatformData(); + IntPtr dllLocal = IntPtr.Zero; + var loader = LibraryLoader.Get(OperatingSystem); if (_PythonDll != "__Internal") { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); + dllLocal = loader.Load(_PythonDll); } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); -#if !(MONO_LINUX || MONO_OSX) if (dllLocal != IntPtr.Zero) { - NativeMethods.FreeLibrary(dllLocal); + loader.Free(dllLocal); } -#endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index a260e8dfa..00a8f0a89 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -6,6 +6,8 @@ namespace Python.Runtime { + using Python.Runtime.Platform; + /// /// The TypeManager class is responsible for building binary-compatible /// Python type objects that are implemented in managed code. @@ -504,15 +506,15 @@ public static NativeCode Active { get { - switch(Runtime.Machine) + switch (Runtime.Machine) { - case Runtime.MachineType.i386: + case MachineType.i386: return I386; - case Runtime.MachineType.x86_64: + case MachineType.x86_64: return X86_64; - case Runtime.MachineType.armv7l: + case MachineType.armv7l: return Armv7l; - case Runtime.MachineType.armv8: + case MachineType.armv8: return Armv8; default: throw new NotImplementedException($"No support for {Runtime.MachineName}"); @@ -635,9 +637,9 @@ int MAP_ANONYMOUS { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: + case OperatingSystemType.Darwin: return 0x1000; - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Linux: return 0x20; default: throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); @@ -668,10 +670,10 @@ internal static IMemoryMapper CreateMemoryMapper() { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: + case OperatingSystemType.Windows: return new WindowsMemoryMapper(); default: throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); From 537ee5fae147c78d0221133f4db4de3371ebc319 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 26 Jun 2019 08:04:00 +0200 Subject: [PATCH 2/2] Implement error handling, move using statements --- src/runtime/platform/LibraryLoader.cs | 115 +++++++++++++++++++------- src/runtime/runtime.cs | 2 +- src/runtime/typemanager.cs | 2 +- 3 files changed, 87 insertions(+), 32 deletions(-) diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index c0157b04b..a6d88cd19 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -1,4 +1,5 @@ using System; +using System.ComponentModel; using System.Runtime.InteropServices; namespace Python.Runtime.Platform @@ -9,7 +10,7 @@ interface ILibraryLoader IntPtr GetFunction(IntPtr hModule, string procedureName); - bool Free(IntPtr hModule); + void Free(IntPtr hModule); } static class LibraryLoader @@ -25,7 +26,7 @@ public static ILibraryLoader Get(OperatingSystemType os) case OperatingSystemType.Linux: return new LinuxLoader(); default: - throw new Exception($"This operating system ({os}) is not supported"); + throw new PlatformNotSupportedException($"This operating system ({os}) is not supported"); } } } @@ -37,15 +38,23 @@ class LinuxLoader : ILibraryLoader private static IntPtr RTLD_DEFAULT = IntPtr.Zero; private const string NativeDll = "libdl.so"; - public IntPtr Load(string fileName) + public IntPtr Load(string dllToLoad) { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); + var filename = $"lib{dllToLoad}.so"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; } - public bool Free(IntPtr handle) + public void Free(IntPtr handle) { dlclose(handle); - return true; } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -56,22 +65,35 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) dllHandle = RTLD_DEFAULT; } - // clear previous errors if any - dlerror(); + ClearError(); IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) + if (res == IntPtr.Zero) { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); } return res; } + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); + public static extern IntPtr dlopen(string fileName, int flags); [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); + private static extern IntPtr dlsym(IntPtr handle, string symbol); [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] private static extern int dlclose(IntPtr handle); @@ -87,15 +109,23 @@ class DarwinLoader : ILibraryLoader private const string NativeDll = "/usr/lib/libSystem.dylib"; private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - public IntPtr Load(string fileName) + public IntPtr Load(string dllToLoad) { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); + var filename = $"lib{dllToLoad}.dylib"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; } - public bool Free(IntPtr handle) + public void Free(IntPtr handle) { dlclose(handle); - return true; } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -106,17 +136,30 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) dllHandle = RTLD_DEFAULT; } - // clear previous errors if any - dlerror(); + ClearError(); IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) + if (res == IntPtr.Zero) { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); } return res; } + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] public static extern IntPtr dlopen(String fileName, int flags); @@ -134,20 +177,32 @@ class WindowsLoader : ILibraryLoader { private const string NativeDll = "kernel32.dll"; - [DllImport(NativeDll)] - static extern IntPtr LoadLibrary(string dllToLoad); - public IntPtr Load(string dllToLoad) => WindowsLoader.LoadLibrary(dllToLoad); + public IntPtr Load(string dllToLoad) + { + var res = WindowsLoader.LoadLibrary(dllToLoad); + if (res == IntPtr.Zero) + throw new DllNotFoundException($"Could not load {dllToLoad}", new Win32Exception()); + return res; + } + + public IntPtr GetFunction(IntPtr hModule, string procedureName) + { + var res = WindowsLoader.GetProcAddress(hModule, procedureName); + if (res == IntPtr.Zero) + throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); + return res; + } - [DllImport(NativeDll)] - static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); - public IntPtr GetFunction(IntPtr hModule, string procedureName) => WindowsLoader.GetProcAddress(hModule, procedureName); + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr LoadLibrary(string dllToLoad); + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); [DllImport(NativeDll)] static extern bool FreeLibrary(IntPtr hModule); - - public bool Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec5bddfd0..a347651d0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -4,10 +4,10 @@ using System.Text; using System.Threading; using System.Collections.Generic; +using Python.Runtime.Platform; namespace Python.Runtime { - using Python.Runtime.Platform; /// /// Encapsulates the low-level Python C API. Note that it is diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 00a8f0a89..127e82eaa 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -3,10 +3,10 @@ using System.Collections.Generic; using System.Reflection; using System.Runtime.InteropServices; +using Python.Runtime.Platform; namespace Python.Runtime { - using Python.Runtime.Platform; /// /// The TypeManager class is responsible for building binary-compatible