diff --git a/AUTHORS.md b/AUTHORS.md index 26285bf6a..3ba87c02a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -29,6 +29,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd2b1dcf..dc644de33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added support for converting python iterators to C# arrays - Changed usage of obselete function GetDelegateForFunctionPointer(IntPtr, Type) to GetDelegateForFunctionPointer(IntPtr) - Added support for kwarg parameters when calling .NET methods from Python +- Added cleanup logic for the managed types on shutdown. ### Fixed diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index ca3c35bfd..26d4b60d8 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -509,7 +509,7 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + [StructLayout(LayoutKind.Sequential)] internal struct Thunk { public Delegate fn; @@ -537,4 +537,13 @@ public ThunkInfo(Delegate target) Address = Marshal.GetFunctionPointerForDelegate(target); } } + + [StructLayout(LayoutKind.Sequential)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 5af2e1a7e..519c3994b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -11,17 +11,30 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; - + private static SlotsHolder _metaSlotsHodler; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } + public static void Release() + { + if (PyCLRMetaType == IntPtr.Zero) + { + throw new ObjectDisposedException("PyCLRMetaType"); + } + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + _metaSlotsHodler.ResetSlots(); + } + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; + } /// /// Metatype __new__ implementation. This is called to create a new diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index bc7500dab..9c235c5f5 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -13,7 +13,6 @@ internal class MethodWrapper public IntPtr mdef; public IntPtr ptr; private bool _disposed = false; - private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 130d90c0a..ed06f40b0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -211,6 +211,7 @@ internal static void Initialize(bool initSigs = false) PyNone = PyObject_GetAttrString(op, "None"); PyTrue = PyObject_GetAttrString(op, "True"); PyFalse = PyObject_GetAttrString(op, "False"); + PySuper_Type = PyObject_GetAttrString(op, "super"); PyBoolType = PyObject_Type(PyTrue); PyNoneType = PyObject_Type(PyNone); @@ -380,6 +381,11 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + + TypeManager.RemoveTypes(); + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + Py_Finalize(); } @@ -401,6 +407,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -1905,6 +1912,22 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); + //==================================================================== // Miscellaneous diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bb920b74f..f677c4c79 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -16,18 +17,38 @@ namespace Python.Runtime /// internal class TypeManager { - private static BindingFlags tbFlags; - private static Dictionary cache; + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static readonly Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); static TypeManager() { - tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); + } public static void Reset() { - cache = new Dictionary(128); + cache.Clear(); + } + + internal static void RemoveTypes() + { + foreach (var tpHandle in cache.Values) + { + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } + } + Runtime.XDecref(tpHandle); + } + cache.Clear(); + _slotsHolders.Clear(); } /// @@ -79,6 +100,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// + /// Return value: Borrowed reference. internal static IntPtr CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name); @@ -90,20 +112,23 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); - + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); InitMethods(type, impl); - return type; } @@ -155,15 +180,18 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + // add a __len__ slot for inheritors of ICollection and ICollection<> if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) { - InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + var method = typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length)); + var thunk = Interop.GetThunk(method); + InitializeSlot(type, thunk, "__len__", slotsHolder); } - // we want to do this after the slot stuff above in case the class itself implements a slot method - InitializeSlots(type, impl.GetType()); - if (base_ != IntPtr.Zero) { Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); @@ -181,12 +209,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = GCHandle.Alloc(impl); @@ -202,12 +234,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) - { - var thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk.Address); - } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -311,7 +337,47 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } - internal static IntPtr CreateMetaType(Type impl) + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + /// + /// Adds a deallocator for a type's method. At deallocation, the deallocator will remove the + /// method from the type's Dict and deallocate the PyMethodDef object. + /// + /// The type to add the deallocator to. + /// The pointer to the PyMethodDef structure. + /// The name of the slot. + /// The SlotsHolder holding the deallocator/. + internal static void AddDeallocator(IntPtr t, IntPtr mdef, string name, SlotsHolder slotsHolder) + { + slotsHolder.AddDealloctor(() => + { + //IntPtr t = type; + IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + } + FreeMethodDef(mdef); + }); + } + + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of @@ -331,8 +397,8 @@ internal static IntPtr CreateMetaType(Type impl) // tp_traverse, tp_clear, tp_is_gc, etc. // Override type slots with those of the managed implementation. - - InitializeSlots(type, impl); + slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default; flags |= TypeFlags.Managed; @@ -343,27 +409,44 @@ internal static IntPtr CreateMetaType(Type impl) // We need space for 3 PyMethodDef structs, each of them // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + Debug.Assert(4 * IntPtr.Size == Marshal.SizeOf(typeof(PyMethodDef))); IntPtr mdefStart = mdef; - ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); + ThunkInfo thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); + slotsHolder.KeepAlive(thunk.Target); + // Add deallocator before writing the method def, as after WriteMethodDef, mdef + // will not have the same value. + AddDeallocator(type, mdef, "__instancecheck__", slotsHolder); mdef = WriteMethodDef( mdef, "__instancecheck__", - thunkInfo.Address + thunk.Address ); - thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + slotsHolder.KeepAlive(thunk.Target); + AddDeallocator(type, mdef, "__subclasscheck__", slotsHolder); + mdef = WriteMethodDef( mdef, "__subclasscheck__", - thunkInfo.Address + thunk.Address ); - // FIXME: mdef is not used + // Pad the last field with zeroes to terminate the array mdef = WriteMethodDefSentinel(mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -403,9 +486,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -684,7 +771,7 @@ internal static void InitializeNativeCodePage() /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { // We work from the most-derived class up; make sure to get // the most-derived slot and not to override it with a base @@ -731,9 +818,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) // These have to be defined, though, so by default we fill these with // static C# functions from this class. - var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; - var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; - + IntPtr ret0 = IntPtr.Zero; + IntPtr ret1 = IntPtr.Zero; if (native != null) { // If we want to support domain reload, the C# implementation @@ -745,6 +831,11 @@ internal static void InitializeSlots(IntPtr type, Type impl) ret1 = NativeCodePage + native.Return1; ret0 = NativeCodePage + native.Return0; } + else + { + ret1 = Interop.GetThunk(((Func)Return1).Method).Address; + ret0 = Interop.GetThunk(((Func)Return0).Method).Address; + } InitializeSlot(type, ret0, "tp_traverse"); InitializeSlot(type, ret0, "tp_clear"); @@ -764,14 +855,46 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// /// Type being initialized. /// Function pointer. - /// Name of the method. + /// Name of the slot to initialize static void InitializeSlot(IntPtr type, IntPtr slot, string name) + { + var offset = GetSlotOffset(name); + if (Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); var offset = (int)fi.GetValue(typeOffset); - Marshal.WriteIntPtr(type, offset, slot); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Add(offset, thunk); + } + } + + static int GetSlotOffset(string name) + { + Type typeOffset = typeof(TypeOffset); + FieldInfo fi = typeOffset.GetField(name); + var offset = (int)fi.GetValue(typeOffset); + return offset; + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } /// @@ -819,5 +942,182 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSlotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + class SlotsHolder + { + /// + /// Delegate called to customize a (Python) Type slot reset + /// + /// The type that will have a slot reset + /// The offset of the slot + public delegate void ResetSlotAction(IntPtr type, int offset); + + private readonly IntPtr type; + private Dictionary slots = new Dictionary(); + private List keepalive = new List(); + private Dictionary customResetors = new Dictionary(); + private List deallocators = new List(); + private bool alreadyReset = false; + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + this.type = type; + } + + public void Add(int offset, ThunkInfo thunk) + { + slots.Add(offset, thunk); + } + + public void Set(int offset, ResetSlotAction resetor) + { + customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + deallocators.Add(deallocate); + } + + /// + /// Add a delegate to keep it from being garbage collected. + /// + /// The delegate to add + public void KeepAlive(Delegate d) + { + keepalive.Add(d); + } + + public void ResetSlots() + { + if (alreadyReset) + { + return; + } + alreadyReset = true; + foreach (var offset in slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); + Marshal.WriteIntPtr(type, offset, ptr); + } + + foreach (var action in deallocators) + { + action(); + } + + foreach (var pair in customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(type, offset); + } + + customResetors.Clear(); + slots.Clear(); + keepalive.Clear(); + deallocators.Clear(); + + // Custom reset + IntPtr tp_base = Marshal.ReadIntPtr(type, TypeOffset.tp_base); + Runtime.XDecref(tp_base); + Marshal.WriteIntPtr(type, TypeOffset.tp_base, IntPtr.Zero); + + IntPtr tp_bases = Marshal.ReadIntPtr(type, TypeOffset.tp_bases); + Runtime.XDecref(tp_bases); + tp_bases = Runtime.PyTuple_New(0); + Marshal.WriteIntPtr(type, TypeOffset.tp_bases, tp_bases); + } + + /// + /// Returns the default C function pointer for the slot to reset. + /// + /// The offset of the slot. + /// The default C function pointer of the slot. + private static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear + || offset == TypeOffset.tp_traverse) + { + return TypeManager.NativeCodePage + TypeManager.NativeCode.Active.Return0; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObject_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = Runtime.PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + Runtime.XDecref(res); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } } }