From 663df73b856b312460fc316ac121e0692dd87141 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 4 May 2020 18:08:56 -0700 Subject: [PATCH 001/151] reworked PythonException to simplify memory management, and enable .NET interop New method ThrowLastAsClrException should be used everywhere instead of obsolete PythonException constructor. It automatically restores .NET exceptions, and applies codecs for Python exceptions. Traceback, PyVal and PyType are now stored and returned as PyObjects. PythonException now has InnerException set from its cause (e.g. __cause__ in Python, if any). PythonException.Restore no longer clears the exception instance. All helper methods were removed from public API surface. --- src/runtime/BorrowedReference.cs | 2 + src/runtime/NewReference.cs | 5 + src/runtime/converterextensions.cs | 2 + src/runtime/exceptions.cs | 5 +- src/runtime/managedtype.cs | 6 + src/runtime/pyobject.cs | 15 +- src/runtime/pythonengine.cs | 17 +- src/runtime/pythonexception.cs | 286 ++++++++++++++++++++--------- src/runtime/runtime.cs | 4 +- 9 files changed, 235 insertions(+), 107 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index a3bf29056..a2af1c7bf 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -14,6 +14,8 @@ readonly ref struct BorrowedReference public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; + public static BorrowedReference Null => new BorrowedReference(); + /// /// Creates new instance of from raw pointer. Unsafe. /// diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 6e66232d0..2e27f2b2b 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -28,6 +28,11 @@ public PyObject MoveToPyObject() return result; } /// + /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + /// + public PyObject MoveToPyObjectOrNull() => this.IsNull() ? null : this.MoveToPyObject(); + /// /// Removes this reference to a Python object, and sets it to null. /// public void Dispose() diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index 667fc6f00..de3067f7a 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -110,6 +110,8 @@ static IPyObjectEncoder[] GetEncoders(Type type) #region Decoding static readonly ConcurrentDictionary pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(BorrowedReference value, BorrowedReference type, Type targetType, out object result) + => TryDecode(value.DangerousGetAddress(), type.DangerousGetAddress(), targetType, out result); internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) { if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 31c367eb2..c7837f7b9 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -279,10 +279,7 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); + pe.Restore(); return; } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..b5fa58042 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -15,6 +15,12 @@ internal abstract class ManagedType internal IntPtr tpHandle; // PyType * + /// + /// Given a Python object, return the associated managed object or null. + /// + internal static ManagedType GetManagedObject(BorrowedReference ob) + => GetManagedObject(ob.DangerousGetAddress()); + /// /// Given a Python object, return the associated managed object or null. /// diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 699ebf873..74e31f7e5 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -31,7 +31,10 @@ public class PyObject : DynamicObject, IEnumerable, IPyDisposable protected internal IntPtr obj = IntPtr.Zero; - internal BorrowedReference Reference => new BorrowedReference(obj); + public static PyObject None => new PyObject(new BorrowedReference(Runtime.PyNone)); + internal BorrowedReference Reference => new BorrowedReference(this.obj); + internal NewReference MakeNewReference() + => NewReference.DangerousFromPointer(Runtime.SelfIncRef(this.obj)); /// /// PyObject Constructor @@ -125,6 +128,13 @@ public static PyObject FromManagedObject(object ob) return new PyObject(op); } + /// + /// Creates new from a nullable reference. + /// When is null, null is returned. + /// + internal static PyObject FromNullableReference(BorrowedReference reference) + => reference.IsNull ? null : new PyObject(reference); + /// /// AsManagedObject Method @@ -226,6 +236,9 @@ public IntPtr[] GetTrackedHandles() return new IntPtr[] { obj }; } + internal BorrowedReference GetPythonTypeReference() + => new BorrowedReference(Runtime.PyObject_TYPE(obj)); + /// /// GetPythonType Method /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 11fc88cd4..18c4698b4 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -755,9 +755,9 @@ public static void With(PyObject obj, Action Body) // Behavior described here: // https://docs.python.org/2/reference/datamodel.html#with-statement-context-managers - IntPtr type = Runtime.PyNone; - IntPtr val = Runtime.PyNone; - IntPtr traceBack = Runtime.PyNone; + PyObject type = PyObject.None; + PyObject val = PyObject.None; + PyObject traceBack = PyObject.None; PythonException ex = null; try @@ -769,15 +769,12 @@ public static void With(PyObject obj, Action Body) catch (PythonException e) { ex = e; - type = ex.PyType.Coalesce(type); - val = ex.PyValue.Coalesce(val); - traceBack = ex.PyTB.Coalesce(traceBack); + type = ex.PyType ?? type; + val = ex.PyValue ?? val; + traceBack = ex.PyTB ?? traceBack; } - Runtime.XIncref(type); - Runtime.XIncref(val); - Runtime.XIncref(traceBack); - var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); + var exitResult = obj.InvokeMethod("__exit__", type, val, traceBack); if (ex != null && !exitResult.IsTrue()) throw ex; } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 8efdccc91..aa4f4df9b 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Text; namespace Python.Runtime @@ -8,78 +9,221 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception, IPyDisposable + public class PythonException : System.Exception, IDisposable { - private IntPtr _pyType = IntPtr.Zero; - private IntPtr _pyValue = IntPtr.Zero; - private IntPtr _pyTB = IntPtr.Zero; - private string _tb = ""; + private PyObject _type; + private PyObject _value; + private PyObject _pyTB; + private string _traceback = ""; private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; - private bool _finalized = false; + [Obsolete("Please, use ThrowLastAsClrException or FromPyErr instead")] public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB); - if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + _type = type.MoveToPyObjectOrNull(); + _value = value.MoveToPyObjectOrNull(); + _pyTB = traceback.MoveToPyObjectOrNull(); + if (_type != null && _value != null) { - string type; string message; - Runtime.XIncref(_pyType); - using (var pyType = new PyObject(_pyType)) - using (PyObject pyTypeName = pyType.GetAttr("__name__")) + using (PyObject pyTypeName = _type.GetAttr("__name__")) { - type = pyTypeName.ToString(); + _pythonTypeName = pyTypeName.ToString(); } - _pythonTypeName = type; + message = _value.ToString(); + _message = _pythonTypeName + " : " + message; + } + if (_pyTB != null) + { + _traceback = TracebackToString(_pyTB); + } + PythonEngine.ReleaseLock(gs); + } + + private PythonException(BorrowedReference pyTypeHandle, + BorrowedReference pyValueHandle, + BorrowedReference pyTracebackHandle, + string message, string pythonTypeName, string traceback, + Exception innerException) + : base(message, innerException) + { + _type = PyObject.FromNullableReference(pyTypeHandle); + _value = PyObject.FromNullableReference(pyValueHandle); + _pyTB = PyObject.FromNullableReference(pyTracebackHandle); + _message = message; + _pythonTypeName = pythonTypeName ?? _pythonTypeName; + _traceback = traceback ?? _traceback; + } - Runtime.XIncref(_pyValue); - using (var pyValue = new PyObject(_pyValue)) + internal static Exception FromPyErr() + { + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + try + { + return FromPyErr( + typeHandle: type, + valueHandle: value, + tracebackHandle: traceback); + } + finally + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + } + } + + internal static Exception FromPyErrOrNull() + { + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + try + { + if (value.IsNull() && type.IsNull() && traceback.IsNull()) { - message = pyValue.ToString(); + return null; } - _message = type + " : " + message; + + var result = FromPyErr(type, value, traceback); + return result; } - if (_pyTB != IntPtr.Zero) + finally + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + } + } + + /// + /// Rethrows the last Python exception as corresponding CLR exception. + /// It is recommended to call this as throw ThrowLastAsClrException() + /// to assist control flow checks. + /// + internal static Exception ThrowLastAsClrException() + { + IntPtr gs = PythonEngine.AcquireLock(); + try { - using (PyObject tb_module = PythonEngine.ImportModule("traceback")) + Runtime.PyErr_Fetch(out var pyTypeHandle, out var pyValueHandle, out var pyTracebackHandle); + try { - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + var clrObject = ManagedType.GetManagedObject(pyValueHandle) as CLRObject; + if (clrObject?.inst is Exception e) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); +#if NETSTANDARD + ExceptionDispatchInfo.Capture(e).Throw(); +#endif + throw e; } + + var result = FromPyErr(pyTypeHandle, pyValueHandle, pyTracebackHandle); + throw result; + } + finally + { + pyTypeHandle.Dispose(); + pyValueHandle.Dispose(); + pyTracebackHandle.Dispose(); } } - PythonEngine.ReleaseLock(gs); + finally + { + PythonEngine.ReleaseLock(gs); + } } - // Ensure that encapsulated Python objects are decref'ed appropriately - // when the managed exception wrapper is garbage-collected. + /// + /// Requires lock to be acquired elsewhere + /// + static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference valueHandle, BorrowedReference tracebackHandle) + { + Exception inner = null; + string pythonTypeName = null, msg = "", traceback = null; + + var clrObject = ManagedType.GetManagedObject(valueHandle) as CLRObject; + if (clrObject?.inst is Exception e) + { + return e; + } + + if (!typeHandle.IsNull && !valueHandle.IsNull) + { + if (PyObjectConversions.TryDecode(valueHandle, typeHandle, typeof(Exception), out object decoded) + && decoded is Exception decodedException) + { + return decodedException; + } + + string type; + string message; + using (var pyType = new PyObject(typeHandle)) + using (PyObject pyTypeName = pyType.GetAttr("__name__")) + { + type = pyTypeName.ToString(); + } + + pythonTypeName = type; + + using (var pyValue = new PyObject(valueHandle)) + { + message = pyValue.ToString(); + var cause = pyValue.GetAttr("__cause__", null); + if (cause != null && cause.Handle != Runtime.PyNone) + { + using (var innerTraceback = cause.GetAttr("__traceback__", null)) + { + inner = FromPyErr( + typeHandle: cause.GetPythonTypeReference(), + valueHandle: cause.Reference, + tracebackHandle: innerTraceback is null + ? BorrowedReference.Null + : innerTraceback.Reference); + } + } + } + msg = type + " : " + message; + } + if (!tracebackHandle.IsNull) + { + traceback = TracebackToString(new PyObject(tracebackHandle)); + } + + return new PythonException(typeHandle, valueHandle, tracebackHandle, + msg, pythonTypeName, traceback, inner); + } - ~PythonException() + static string TracebackToString(PyObject traceback) { - if (_finalized || disposed) + if (traceback is null) { - return; + throw new ArgumentNullException(nameof(traceback)); } - _finalized = true; - Finalizer.Instance.AddFinalizedObject(this); + + PyObject tracebackModule = PythonEngine.ImportModule("traceback"); + PyList stackLines = new PyList(tracebackModule.InvokeMethod("format_tb", traceback)); + stackLines.Reverse(); + var result = new StringBuilder(); + foreach (object stackLine in stackLines) + { + result.Append(stackLine); + } + return result.ToString(); } /// - /// Restores python error. + /// Restores python error. Clears this instance. /// public void Restore() { + if (this.disposed) throw new ObjectDisposedException(nameof(PythonException)); + IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Restore(_pyType, _pyValue, _pyTB); - _pyType = IntPtr.Zero; - _pyValue = IntPtr.Zero; - _pyTB = IntPtr.Zero; + Runtime.PyErr_Restore(_type.MakeNewReference(), _value.MakeNewReference(), _pyTB.MakeNewReference()); PythonEngine.ReleaseLock(gs); } @@ -89,10 +233,7 @@ public void Restore() /// /// Returns the exception type as a Python object. /// - public IntPtr PyType - { - get { return _pyType; } - } + public PyObject PyType => _type; /// /// PyValue Property @@ -100,10 +241,7 @@ public IntPtr PyType /// /// Returns the exception value as a Python object. /// - public IntPtr PyValue - { - get { return _pyValue; } - } + public PyObject PyValue => _value; /// /// PyTB Property @@ -111,10 +249,7 @@ public IntPtr PyValue /// /// Returns the TraceBack as a Python object. /// - public IntPtr PyTB - { - get { return _pyTB; } - } + public PyObject PyTB => _pyTB; /// /// Message Property @@ -135,7 +270,7 @@ public override string Message /// public override string StackTrace { - get { return _tb + base.StackTrace; } + get { return _traceback + base.StackTrace; } } /// @@ -156,18 +291,12 @@ public string Format() IntPtr gs = PythonEngine.AcquireLock(); try { - if (_pyTB != IntPtr.Zero && _pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + if (_pyTB != null && _type != null && _value != null) { - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); - using (PyObject pyType = new PyObject(_pyType)) - using (PyObject pyValue = new PyObject(_pyValue)) - using (PyObject pyTB = new PyObject(_pyTB)) using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) { var buffer = new StringBuilder(); - var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); + var values = tb_mod.InvokeMethod("format_exception", _type, _value, _pyTB); foreach (PyObject val in values) { buffer.Append(val.ToString()); @@ -200,39 +329,14 @@ public void Dispose() { if (!disposed) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) - { - IntPtr gs = PythonEngine.AcquireLock(); - if (_pyType != IntPtr.Zero) - { - Runtime.XDecref(_pyType); - _pyType= IntPtr.Zero; - } - - if (_pyValue != IntPtr.Zero) - { - Runtime.XDecref(_pyValue); - _pyValue = IntPtr.Zero; - } - - // XXX Do we ever get TraceBack? // - if (_pyTB != IntPtr.Zero) - { - Runtime.XDecref(_pyTB); - _pyTB = IntPtr.Zero; - } - PythonEngine.ReleaseLock(gs); - } + _type?.Dispose(); + _value?.Dispose(); + _pyTB?.Dispose(); GC.SuppressFinalize(this); disposed = true; } } - public IntPtr[] GetTrackedHandles() - { - return new IntPtr[] { _pyType, _pyValue, _pyTB }; - } - /// /// Matches Method /// @@ -240,24 +344,26 @@ public IntPtr[] GetTrackedHandles() /// Returns true if the Python exception type represented by the /// PythonException instance matches the given exception type. /// - public static bool Matches(IntPtr ob) + internal static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } - public static void ThrowIfIsNull(IntPtr ob) + internal static IntPtr ThrowIfIsNull(IntPtr ob) { if (ob == IntPtr.Zero) { - throw new PythonException(); + throw ThrowLastAsClrException(); } + + return ob; } - public static void ThrowIfIsNotZero(int value) + internal static void ThrowIfIsNotZero(int value) { if (value != 0) { - throw new PythonException(); + throw ThrowLastAsClrException(); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 3d2610fea..e9a020ed2 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1955,10 +1955,10 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern IntPtr PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); + internal static extern void PyErr_Fetch(out NewReference type, out NewReference value, out NewReference traceback); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); + internal static extern void PyErr_Restore(NewReference type, NewReference value, NewReference traceback); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Clear(); From d45c39adf05db60982284bcade992ac7f3e374d8 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 24 Feb 2020 17:22:28 -0800 Subject: [PATCH 002/151] allows codecs to encode and decode thrown exceptions --- src/embed_tests/Codecs.cs | 53 +++++++++++++++++++ .../Python.EmbeddingTest.15.csproj | 2 +- src/runtime/exceptions.cs | 2 +- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 18fcd32d1..0cd723338 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -82,6 +82,59 @@ static void TupleRoundtripGeneric() { Assert.AreEqual(expected: tuple, actual: restored); } } + + const string TestExceptionMessage = "Hello World!"; + [Test] + public void ExceptionEncoded() { + PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); + void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); + var callMeAction = new Action(CallMe); + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + scope.Exec(@" +def call(func): + try: + func() + except ValueError as e: + return str(e) +"); + var callFunc = scope.Get("call"); + string message = callFunc.Invoke(callMeAction.ToPython()).As(); + Assert.AreEqual(TestExceptionMessage, message); + } + + [Test] + public void ExceptionDecoded() { + PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() => PythonEngine.Exec( + $"raise ValueError('{TestExceptionMessage}')")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + + class ValueErrorWrapper : Exception { + public ValueErrorWrapper(string message) : base(message) { } + } + + class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder { + public bool CanDecode(PyObject objectType, Type targetType) + => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError")); + + public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) + || typeof(ValueErrorWrapper).IsSubclassOf(type); + + public bool TryDecode(PyObject pyObj, out T value) { + var message = pyObj.GetAttr("args")[0].As(); + value = (T)(object)new ValueErrorWrapper(message); + return true; + } + + public PyObject TryEncode(object value) { + var error = (ValueErrorWrapper)value; + return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); + } + } } /// diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index e163c9242..9757c9294 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 7.3 + 8.0 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index c7837f7b9..336362685 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -283,7 +283,7 @@ public static void SetError(Exception e) return; } - IntPtr op = CLRObject.GetInstHandle(e); + IntPtr op = Converter.ToPython(e); IntPtr etype = Runtime.PyObject_GetAttrString(op, "__class__"); Runtime.PyErr_SetObject(etype, op); Runtime.XDecref(etype); From 0d941c576bc676c718bac189d228aa261018c307 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 4 May 2020 18:27:01 -0700 Subject: [PATCH 003/151] turned on CLR exception marshaling --- src/runtime/delegatemanager.cs | 3 +-- src/runtime/exceptions.cs | 4 ++-- src/runtime/pydict.cs | 12 ++++++------ src/runtime/pyiter.cs | 2 +- src/runtime/pylist.cs | 14 ++++++------- src/runtime/pyobject.cs | 36 +++++++++++++++++----------------- src/runtime/pyscope.cs | 16 +++++++-------- src/runtime/pysequence.cs | 12 ++++++------ src/runtime/pythonengine.cs | 2 +- src/runtime/runtime.cs | 2 +- 10 files changed, 51 insertions(+), 52 deletions(-) diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index bd8f1ee4c..7dffe422b 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -257,8 +257,7 @@ public object TrueDispatch(ArrayList args) if (op == IntPtr.Zero) { - var e = new PythonException(); - throw e; + throw PythonException.ThrowLastAsClrException(); } if (rtype == typeof(void)) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 336362685..cb3089fe3 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -197,7 +197,7 @@ internal static void ErrorCheck(IntPtr pointer) { if (pointer == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -208,7 +208,7 @@ internal static void ErrorOccurredCheck(IntPtr pointer) { if (pointer == IntPtr.Zero || ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 7ff7a83c8..116d60e39 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -34,7 +34,7 @@ public PyDict() obj = Runtime.PyDict_New(); if (obj == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -108,7 +108,7 @@ public PyObject Keys() IntPtr items = Runtime.PyDict_Keys(obj); if (items == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(items); } @@ -125,7 +125,7 @@ public PyObject Values() IntPtr items = Runtime.PyDict_Values(obj); if (items == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(items); } @@ -144,7 +144,7 @@ public PyObject Items() { if (items.IsNull()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return items.MoveToPyObject(); @@ -167,7 +167,7 @@ public PyDict Copy() IntPtr op = Runtime.PyDict_Copy(obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyDict(op); } @@ -184,7 +184,7 @@ public void Update(PyObject other) int result = Runtime.PyDict_Update(obj, other.obj); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index ee07bcecf..5b0f42bfa 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -36,7 +36,7 @@ public PyIter(PyObject iterable) obj = Runtime.PyObject_GetIter(iterable.obj); if (obj == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 347cc3000..40e162c6a 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -53,7 +53,7 @@ public PyList() obj = Runtime.PyList_New(0); if (obj == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -75,7 +75,7 @@ public PyList(PyObject[] items) int r = Runtime.PyList_SetItem(obj, i, ptr); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -106,7 +106,7 @@ public static PyList AsList(PyObject value) IntPtr op = Runtime.PySequence_List(value.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyList(op); } @@ -123,7 +123,7 @@ public void Append(PyObject item) int r = Runtime.PyList_Append(this.Reference, item.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -138,7 +138,7 @@ public void Insert(int index, PyObject item) int r = Runtime.PyList_Insert(this.Reference, index, item.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -154,7 +154,7 @@ public void Reverse() int r = Runtime.PyList_Reverse(this.Reference); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -170,7 +170,7 @@ public void Sort() int r = Runtime.PyList_Sort(this.Reference); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 74e31f7e5..3b5edf73e 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -311,7 +311,7 @@ public PyObject GetAttr(string name) IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -353,7 +353,7 @@ public PyObject GetAttr(PyObject name) IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -396,7 +396,7 @@ public void SetAttr(string name, PyObject value) int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -417,7 +417,7 @@ public void SetAttr(PyObject name, PyObject value) int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -436,7 +436,7 @@ public void DelAttr(string name) int r = Runtime.PyObject_SetAttrString(obj, name, IntPtr.Zero); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -456,7 +456,7 @@ public void DelAttr(PyObject name) int r = Runtime.PyObject_SetAttr(obj, name.obj, IntPtr.Zero); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -476,7 +476,7 @@ public virtual PyObject GetItem(PyObject key) IntPtr op = Runtime.PyObject_GetItem(obj, key.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -534,7 +534,7 @@ public virtual void SetItem(PyObject key, PyObject value) int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -593,7 +593,7 @@ public virtual void DelItem(PyObject key) int r = Runtime.PyObject_DelItem(obj, key.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -708,7 +708,7 @@ public PyObject GetIterator() IntPtr r = Runtime.PyObject_GetIter(obj); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -744,7 +744,7 @@ public PyObject Invoke(params PyObject[] args) t.Dispose(); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -764,7 +764,7 @@ public PyObject Invoke(PyTuple args) IntPtr r = Runtime.PyObject_Call(obj, args.obj, IntPtr.Zero); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -787,7 +787,7 @@ public PyObject Invoke(PyObject[] args, PyDict kw) t.Dispose(); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -807,7 +807,7 @@ public PyObject Invoke(PyTuple args, PyDict kw) IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw?.obj ?? IntPtr.Zero); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -1028,7 +1028,7 @@ public PyList Dir() IntPtr r = Runtime.PyObject_Dir(obj); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyList(r); } @@ -1086,7 +1086,7 @@ public override bool Equals(object o) int r = Runtime.PyObject_Compare(obj, ((PyObject)o).obj); if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return r == 0; } @@ -1127,7 +1127,7 @@ public override bool TrySetMember(SetMemberBinder binder, object value) int r = Runtime.PyObject_SetAttrString(obj, binder.Name, ptr); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } Runtime.XDecref(ptr); return true; @@ -1199,7 +1199,7 @@ private static void AddArgument(IntPtr argtuple, int i, object target) if (Runtime.PyTuple_SetItem(argtuple, i, ptr) < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8738824f5..b22f7d213 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -186,7 +186,7 @@ public void ImportAll(PyScope scope) int result = Runtime.PyDict_Update(variables, scope.variables); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -206,7 +206,7 @@ public void ImportAll(PyObject module) int result = Runtime.PyDict_Update(variables, module_dict); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -221,7 +221,7 @@ public void ImportAll(PyDict dict) int result = Runtime.PyDict_Update(variables, dict.obj); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -333,7 +333,7 @@ private void Exec(string code, IntPtr _globals, IntPtr _locals) Runtime.CheckExceptionOccurred(); if (!reference.IsNone()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } finally @@ -364,7 +364,7 @@ private void Set(string name, IntPtr value) int r = Runtime.PyObject_SetItem(variables, pyKey.obj, value); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -383,7 +383,7 @@ public void Remove(string name) int r = Runtime.PyObject_DelItem(variables, pyKey.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -438,7 +438,7 @@ public bool TryGet(string name, out PyObject value) IntPtr op = Runtime.PyObject_GetItem(variables, pyKey.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } if (op == Runtime.PyNone) { @@ -577,7 +577,7 @@ internal PyScope NewScope(string name) var module = Runtime.PyModule_New(name); if (module == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyScope(module, this); } diff --git a/src/runtime/pysequence.cs b/src/runtime/pysequence.cs index bfaee79a6..b1e89b835 100644 --- a/src/runtime/pysequence.cs +++ b/src/runtime/pysequence.cs @@ -44,7 +44,7 @@ public PyObject GetSlice(int i1, int i2) IntPtr op = Runtime.PySequence_GetSlice(obj, i1, i2); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -61,7 +61,7 @@ public void SetSlice(int i1, int i2, PyObject v) int r = Runtime.PySequence_SetSlice(obj, i1, i2, v.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -77,7 +77,7 @@ public void DelSlice(int i1, int i2) int r = Runtime.PySequence_DelSlice(obj, i1, i2); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -113,7 +113,7 @@ public bool Contains(PyObject item) int r = Runtime.PySequence_Contains(obj, item.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return r != 0; } @@ -131,7 +131,7 @@ public PyObject Concat(PyObject other) IntPtr op = Runtime.PySequence_Concat(obj, other.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -149,7 +149,7 @@ public PyObject Repeat(int count) IntPtr op = Runtime.PySequence_Repeat(obj, count); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 18c4698b4..6e0716166 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -548,7 +548,7 @@ public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = nu { if (result.obj != Runtime.PyNone) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e9a020ed2..84c315245 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -494,7 +494,7 @@ internal static void CheckExceptionOccurred() { if (PyErr_Occurred() != IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } From 02cd234a734ea550565aff953602f4f1978ae565 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 4 May 2020 20:17:37 -0700 Subject: [PATCH 004/151] revert to C# 7.3 --- src/embed_tests/Codecs.cs | 26 +++++++++++-------- .../Python.EmbeddingTest.15.csproj | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 0cd723338..038510951 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -89,28 +89,32 @@ public void ExceptionEncoded() { PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); var callMeAction = new Action(CallMe); - using var _ = Py.GIL(); - using var scope = Py.CreateScope(); - scope.Exec(@" + using (var _ = Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(@" def call(func): try: func() except ValueError as e: return str(e) "); - var callFunc = scope.Get("call"); - string message = callFunc.Invoke(callMeAction.ToPython()).As(); - Assert.AreEqual(TestExceptionMessage, message); + var callFunc = scope.Get("call"); + string message = callFunc.Invoke(callMeAction.ToPython()).As(); + Assert.AreEqual(TestExceptionMessage, message); + } } [Test] public void ExceptionDecoded() { PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); - using var _ = Py.GIL(); - using var scope = Py.CreateScope(); - var error = Assert.Throws(() => PythonEngine.Exec( - $"raise ValueError('{TestExceptionMessage}')")); - Assert.AreEqual(TestExceptionMessage, error.Message); + using (var _ = Py.GIL()) + using (var scope = Py.CreateScope()) + { + var error = Assert.Throws(() + => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } } class ValueErrorWrapper : Exception { diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 9757c9294..e163c9242 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 8.0 + 7.3 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT From defb200955d0789aa68c82ce6fb3714408ca660e Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 4 May 2020 22:07:57 -0700 Subject: [PATCH 005/151] fixed Restore when not all exception fields are set --- src/runtime/pyobject.cs | 5 +++-- src/runtime/pythonexception.cs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b5edf73e..6986256a4 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -33,8 +33,9 @@ public class PyObject : DynamicObject, IEnumerable, IPyDisposable public static PyObject None => new PyObject(new BorrowedReference(Runtime.PyNone)); internal BorrowedReference Reference => new BorrowedReference(this.obj); - internal NewReference MakeNewReference() - => NewReference.DangerousFromPointer(Runtime.SelfIncRef(this.obj)); + internal NewReference MakeNewReferenceOrNull() + => NewReference.DangerousFromPointer( + this.obj == IntPtr.Zero ? IntPtr.Zero : Runtime.SelfIncRef(this.obj)); /// /// PyObject Constructor diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index aa4f4df9b..d0523927a 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -223,7 +223,10 @@ public void Restore() if (this.disposed) throw new ObjectDisposedException(nameof(PythonException)); IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Restore(_type.MakeNewReference(), _value.MakeNewReference(), _pyTB.MakeNewReference()); + Runtime.PyErr_Restore( + _type.MakeNewReferenceOrNull(), + _value.MakeNewReferenceOrNull(), + _pyTB.MakeNewReferenceOrNull()); PythonEngine.ReleaseLock(gs); } From 945dc85989ddd76a40245a53b3dbcd47cc3b2f4b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 4 May 2020 22:08:18 -0700 Subject: [PATCH 006/151] cleaned PythonException code up a bit --- src/runtime/pythonexception.cs | 67 +++++++++++++++------------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index d0523927a..914f421ef 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -29,14 +29,12 @@ public PythonException() _pyTB = traceback.MoveToPyObjectOrNull(); if (_type != null && _value != null) { - string message; using (PyObject pyTypeName = _type.GetAttr("__name__")) { _pythonTypeName = pyTypeName.ToString(); } - message = _value.ToString(); - _message = _pythonTypeName + " : " + message; + _message = _pythonTypeName + " : " + _value; } if (_pyTB != null) { @@ -45,19 +43,17 @@ public PythonException() PythonEngine.ReleaseLock(gs); } - private PythonException(BorrowedReference pyTypeHandle, - BorrowedReference pyValueHandle, - BorrowedReference pyTracebackHandle, - string message, string pythonTypeName, string traceback, + private PythonException(PyObject type, PyObject value, PyObject traceback, + string message, string pythonTypeName, string tracebackText, Exception innerException) : base(message, innerException) { - _type = PyObject.FromNullableReference(pyTypeHandle); - _value = PyObject.FromNullableReference(pyValueHandle); - _pyTB = PyObject.FromNullableReference(pyTracebackHandle); + _type = type; + _value = value; + _pyTB = traceback; _message = message; _pythonTypeName = pythonTypeName ?? _pythonTypeName; - _traceback = traceback ?? _traceback; + _traceback = tracebackText ?? _traceback; } internal static Exception FromPyErr() @@ -143,7 +139,7 @@ internal static Exception ThrowLastAsClrException() static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference valueHandle, BorrowedReference tracebackHandle) { Exception inner = null; - string pythonTypeName = null, msg = "", traceback = null; + string pythonTypeName = null, msg = "", tracebackText = null; var clrObject = ManagedType.GetManagedObject(valueHandle) as CLRObject; if (clrObject?.inst is Exception e) @@ -151,7 +147,11 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value return e; } - if (!typeHandle.IsNull && !valueHandle.IsNull) + var type = PyObject.FromNullableReference(typeHandle); + var value = PyObject.FromNullableReference(valueHandle); + var traceback = PyObject.FromNullableReference(tracebackHandle); + + if (type != null && value != null) { if (PyObjectConversions.TryDecode(valueHandle, typeHandle, typeof(Exception), out object decoded) && decoded is Exception decodedException) @@ -159,42 +159,33 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value return decodedException; } - string type; - string message; - using (var pyType = new PyObject(typeHandle)) - using (PyObject pyTypeName = pyType.GetAttr("__name__")) + using (PyObject pyTypeName = type.GetAttr("__name__")) { - type = pyTypeName.ToString(); + pythonTypeName = pyTypeName.ToString(); } - pythonTypeName = type; - - using (var pyValue = new PyObject(valueHandle)) + var cause = value.GetAttr("__cause__", null); + if (cause != null && cause.Handle != Runtime.PyNone) { - message = pyValue.ToString(); - var cause = pyValue.GetAttr("__cause__", null); - if (cause != null && cause.Handle != Runtime.PyNone) + using (var innerTraceback = cause.GetAttr("__traceback__", null)) { - using (var innerTraceback = cause.GetAttr("__traceback__", null)) - { - inner = FromPyErr( - typeHandle: cause.GetPythonTypeReference(), - valueHandle: cause.Reference, - tracebackHandle: innerTraceback is null - ? BorrowedReference.Null - : innerTraceback.Reference); - } + inner = FromPyErr( + typeHandle: cause.GetPythonTypeReference(), + valueHandle: cause.Reference, + tracebackHandle: innerTraceback is null + ? BorrowedReference.Null + : innerTraceback.Reference); } } - msg = type + " : " + message; + msg = pythonTypeName + " : " + value; } - if (!tracebackHandle.IsNull) + if (traceback != null) { - traceback = TracebackToString(new PyObject(tracebackHandle)); + tracebackText = TracebackToString(traceback); } - return new PythonException(typeHandle, valueHandle, tracebackHandle, - msg, pythonTypeName, traceback, inner); + return new PythonException(type, value, traceback, + msg, pythonTypeName, tracebackText, inner); } static string TracebackToString(PyObject traceback) From 3d95e60fa9282d746355e0223a919a57f5a32279 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 5 May 2020 00:05:29 -0700 Subject: [PATCH 007/151] remove unused overloads of FromPyErr --- src/runtime/pythonexception.cs | 39 ---------------------------------- 1 file changed, 39 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 914f421ef..dd308f3c5 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -56,45 +56,6 @@ private PythonException(PyObject type, PyObject value, PyObject traceback, _traceback = tracebackText ?? _traceback; } - internal static Exception FromPyErr() - { - Runtime.PyErr_Fetch(out var type, out var value, out var traceback); - try - { - return FromPyErr( - typeHandle: type, - valueHandle: value, - tracebackHandle: traceback); - } - finally - { - type.Dispose(); - value.Dispose(); - traceback.Dispose(); - } - } - - internal static Exception FromPyErrOrNull() - { - Runtime.PyErr_Fetch(out var type, out var value, out var traceback); - try - { - if (value.IsNull() && type.IsNull() && traceback.IsNull()) - { - return null; - } - - var result = FromPyErr(type, value, traceback); - return result; - } - finally - { - type.Dispose(); - value.Dispose(); - traceback.Dispose(); - } - } - /// /// Rethrows the last Python exception as corresponding CLR exception. /// It is recommended to call this as throw ThrowLastAsClrException() From d6679987886b82e30f01698a32165bdb54269753 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 5 May 2020 00:06:51 -0700 Subject: [PATCH 008/151] capture exception on Exceptions.SetError, restore in ThrowLastAsClrException --- src/runtime/exceptions.cs | 7 +++++++ src/runtime/pythonexception.cs | 29 ++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index cb3089fe3..d450c8137 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; namespace Python.Runtime @@ -285,6 +286,12 @@ public static void SetError(Exception e) IntPtr op = Converter.ToPython(e); IntPtr etype = Runtime.PyObject_GetAttrString(op, "__class__"); +#if NETSTANDARD + var exceptionInfo = ExceptionDispatchInfo.Capture(e); + Runtime.XDecref(op); + op = Converter.ToPython(exceptionInfo); +#endif + Runtime.PyErr_SetObject(etype, op); Runtime.XDecref(etype); Runtime.XDecref(op); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index dd308f3c5..f917bbaf6 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -66,26 +66,30 @@ internal static Exception ThrowLastAsClrException() IntPtr gs = PythonEngine.AcquireLock(); try { - Runtime.PyErr_Fetch(out var pyTypeHandle, out var pyValueHandle, out var pyTracebackHandle); + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); try { - var clrObject = ManagedType.GetManagedObject(pyValueHandle) as CLRObject; - if (clrObject?.inst is Exception e) - { + var clrObject = ManagedType.GetManagedObject(value) as CLRObject; #if NETSTANDARD - ExceptionDispatchInfo.Capture(e).Throw(); + if (clrObject?.inst is ExceptionDispatchInfo storedException) + { + storedException.Throw(); + throw storedException.SourceException; // unreachable + } #endif + if (clrObject?.inst is Exception e) + { throw e; } - var result = FromPyErr(pyTypeHandle, pyValueHandle, pyTracebackHandle); + var result = FromPyErr(type, value, traceback); throw result; } finally { - pyTypeHandle.Dispose(); - pyValueHandle.Dispose(); - pyTracebackHandle.Dispose(); + type.Dispose(); + value.Dispose(); + traceback.Dispose(); } } finally @@ -108,6 +112,13 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value return e; } +#if NETSTANDARD + if (clrObject?.inst is ExceptionDispatchInfo exceptionDispatchInfo) + { + return exceptionDispatchInfo.SourceException; + } +#endif + var type = PyObject.FromNullableReference(typeHandle); var value = PyObject.FromNullableReference(valueHandle); var traceback = PyObject.FromNullableReference(tracebackHandle); From bec8b7dfb83a09c96b3083b7188105101cbce46b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 5 May 2020 00:39:01 -0700 Subject: [PATCH 009/151] fixed failure in ExceptionEncoded test case caused by ExceptionDispatchInfo masking exception object --- src/runtime/converter.cs | 2 ++ src/runtime/exceptions.cs | 6 ++-- src/runtime/pythonexception.cs | 59 +++++++++++++++++++++++++++------- src/runtime/runtime.cs | 3 ++ 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 3add8aba0..d4c407214 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -293,6 +293,8 @@ internal static bool ToManaged(IntPtr value, Type type, } + internal static bool ToManagedValue(BorrowedReference value, Type obType, out object result, bool setError) + => ToManagedValue(value.DangerousGetAddress(), obType, out result, setError); internal static bool ToManagedValue(IntPtr value, Type obType, out object result, bool setError) { diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index d450c8137..3d8d66b29 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -262,6 +262,7 @@ public static void SetError(IntPtr ob, IntPtr value) Runtime.PyErr_SetObject(ob, value); } + internal const string DispatchInfoAttribute = "__dispatch_info__"; /// /// SetError Method /// @@ -288,8 +289,9 @@ public static void SetError(Exception e) IntPtr etype = Runtime.PyObject_GetAttrString(op, "__class__"); #if NETSTANDARD var exceptionInfo = ExceptionDispatchInfo.Capture(e); - Runtime.XDecref(op); - op = Converter.ToPython(exceptionInfo); + IntPtr pyInfo = Converter.ToPython(exceptionInfo); + Runtime.PyObject_SetAttrString(op, DispatchInfoAttribute, pyInfo); + Runtime.XDecref(pyInfo); #endif Runtime.PyErr_SetObject(etype, op); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index f917bbaf6..7f28fdb51 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -69,14 +69,19 @@ internal static Exception ThrowLastAsClrException() Runtime.PyErr_Fetch(out var type, out var value, out var traceback); try { - var clrObject = ManagedType.GetManagedObject(value) as CLRObject; #if NETSTANDARD - if (clrObject?.inst is ExceptionDispatchInfo storedException) + if (!value.IsNull()) { - storedException.Throw(); - throw storedException.SourceException; // unreachable + var exceptionInfo = TryGetDispatchInfo(value); + if (exceptionInfo != null) + { + exceptionInfo.Throw(); + throw exceptionInfo.SourceException; // unreachable + } } #endif + + var clrObject = ManagedType.GetManagedObject(value) as CLRObject; if (clrObject?.inst is Exception e) { throw e; @@ -98,6 +103,37 @@ internal static Exception ThrowLastAsClrException() } } +#if NETSTANDARD + static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference exception) + { + if (exception.IsNull) return null; + + var pyInfo = Runtime.PyObject_GetAttrString(exception, Exceptions.DispatchInfoAttribute); + if (pyInfo.IsNull()) + { + if (Exceptions.ExceptionMatches(Exceptions.AttributeError)) + { + Exceptions.Clear(); + } + return null; + } + + try + { + if (Converter.ToManagedValue(pyInfo, typeof(ExceptionDispatchInfo), out object result, setError: false)) + { + return (ExceptionDispatchInfo)result; + } + + return null; + } + finally + { + pyInfo.Dispose(); + } + } +#endif + /// /// Requires lock to be acquired elsewhere /// @@ -106,19 +142,20 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value Exception inner = null; string pythonTypeName = null, msg = "", tracebackText = null; - var clrObject = ManagedType.GetManagedObject(valueHandle) as CLRObject; - if (clrObject?.inst is Exception e) - { - return e; - } - #if NETSTANDARD - if (clrObject?.inst is ExceptionDispatchInfo exceptionDispatchInfo) + var exceptionDispatchInfo = TryGetDispatchInfo(valueHandle); + if (exceptionDispatchInfo != null) { return exceptionDispatchInfo.SourceException; } #endif + var clrObject = ManagedType.GetManagedObject(valueHandle) as CLRObject; + if (clrObject?.inst is Exception e) + { + return e; + } + var type = PyObject.FromNullableReference(typeHandle); var value = PyObject.FromNullableReference(valueHandle); var traceback = PyObject.FromNullableReference(tracebackHandle); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 84c315245..d7f090cb2 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -937,7 +937,10 @@ internal static bool PyObject_IsIterable(IntPtr pointer) internal static extern int PyObject_HasAttrString(IntPtr pointer, string name); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + [Obsolete("Use overload accepting BorrowedReference")] internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyObject_GetAttrString(BorrowedReference pointer, string name); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value); From 0961c94a573586a2eb1a103caf87579712c7ad2d Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 6 May 2020 14:16:04 -0700 Subject: [PATCH 010/151] added tests for __cause__/InnerException #893 #1098 --- src/embed_tests/TestPythonException.cs | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 000c32ca3..65cc18004 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -55,6 +55,36 @@ public void TestPythonErrorTypeName() } } + [Test] + public void TestNestedExceptions() + { + try + { + PythonEngine.Exec(@" +try: + raise Exception('inner') +except Exception as ex: + raise Exception('outer') from ex +"); + } + catch (PythonException ex) + { + Assert.That(ex.InnerException, Is.InstanceOf()); + Assert.That(ex.InnerException.Message, Is.EqualTo("Exception : inner")); + } + } + + [Test] + public void InnerIsEmptyWithNoCause() + { + var list = new PyList(); + PyObject foo = null; + + var ex = Assert.Throws(() => foo = list[0]); + + Assert.IsNull(ex.InnerException); + } + [Test] public void TestPythonExceptionFormat() { From 2cd8627703a764655ba6e6f8b5525c9ca5f6eb09 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 13 May 2020 21:18:14 -0700 Subject: [PATCH 011/151] introduced StolenReference type The new type indicates parameters of C API functions, that steal references to passed objects. --- src/runtime/NewReference.cs | 10 ++++++++++ src/runtime/Python.Runtime.csproj | 1 + src/runtime/StolenReference.cs | 18 ++++++++++++++++++ src/runtime/pyobject.cs | 2 +- src/runtime/pythonexception.cs | 6 +++--- src/runtime/runtime.cs | 2 +- 6 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 src/runtime/StolenReference.cs diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 2e27f2b2b..7abf0b708 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -33,6 +33,16 @@ public PyObject MoveToPyObject() /// public PyObject MoveToPyObjectOrNull() => this.IsNull() ? null : this.MoveToPyObject(); /// + /// Call this method to move ownership of this reference to a Python C API function, + /// that steals reference passed to it. + /// + public StolenReference Steal() + { + IntPtr rawPointer = this.pointer; + this.pointer = IntPtr.Zero; + return new StolenReference(rawPointer); + } + /// /// Removes this reference to a Python object, and sets it to null. /// public void Dispose() diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 2e47805cb..0dc46bd53 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -146,6 +146,7 @@ + diff --git a/src/runtime/StolenReference.cs b/src/runtime/StolenReference.cs new file mode 100644 index 000000000..fb789eec5 --- /dev/null +++ b/src/runtime/StolenReference.cs @@ -0,0 +1,18 @@ +namespace Python.Runtime +{ + using System; + + /// + /// Should only be used for the arguments of Python C API functions, that steal references + /// + [NonCopyable] + readonly ref struct StolenReference + { + readonly IntPtr pointer; + + internal StolenReference(IntPtr pointer) + { + this.pointer = pointer; + } + } +} diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 6986256a4..822cb4496 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -215,7 +215,7 @@ protected virtual void Dispose(bool disposing) { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType, errVal, traceback); + Runtime.PyErr_Restore(errType.Steal(), errVal.Steal(), traceback.Steal()); } } else diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 7f28fdb51..a3c1df93b 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -224,9 +224,9 @@ public void Restore() IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Restore( - _type.MakeNewReferenceOrNull(), - _value.MakeNewReferenceOrNull(), - _pyTB.MakeNewReferenceOrNull()); + _type.MakeNewReferenceOrNull().Steal(), + _value.MakeNewReferenceOrNull().Steal(), + _pyTB.MakeNewReferenceOrNull().Steal()); PythonEngine.ReleaseLock(gs); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d7f090cb2..0a10d0e91 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1961,7 +1961,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern void PyErr_Fetch(out NewReference type, out NewReference value, out NewReference traceback); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Restore(NewReference type, NewReference value, NewReference traceback); + internal static extern void PyErr_Restore(StolenReference type, StolenReference value, StolenReference traceback); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Clear(); From e4c1b9be57ae9fe670c15189648a87277e41d923 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 28 May 2020 18:17:19 -0700 Subject: [PATCH 012/151] prevent undebuggable StackOverflowException during exception interop --- src/runtime/pythonexception.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index a3c1df93b..d544551e0 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -63,6 +63,10 @@ private PythonException(PyObject type, PyObject value, PyObject traceback, /// internal static Exception ThrowLastAsClrException() { + // prevent potential interop errors in this method + // from crashing process with undebuggable StackOverflowException + RuntimeHelpers.EnsureSufficientExecutionStack(); + IntPtr gs = PythonEngine.AcquireLock(); try { From 3c0b737030466d684293d4537a10356dcf3af778 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 22 Jun 2020 13:49:35 -0700 Subject: [PATCH 013/151] README: added summary of new exception handling features --- CHANGELOG.md | 5 +++++ pythonnet.15.sln | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afb2badbd..79e1bf0be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Improved exception handling: + - exceptions can now be converted with codecs + - `InnerException` and `__cause__` are propagated properly + - .NET and Python exceptions are preserved when crossing Python/.NET boundary + ### Changed ### Fixed diff --git a/pythonnet.15.sln b/pythonnet.15.sln index ce863817f..1105f1afb 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -39,7 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recip ProjectSection(SolutionItems) = preProject conda.recipe\bld.bat = conda.recipe\bld.bat conda.recipe\meta.yaml = conda.recipe\meta.yaml - conda.recipe\README.md = conda.recipe\README.md EndProjectSection EndProject Global From 0f33f71fa14c688610d5225691c6b6777a4bf29b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 31 Dec 2020 13:17:25 -0800 Subject: [PATCH 014/151] disable implicit conversion from PyFloat to .NET integer types (fixes #1342) --- CHANGELOG.md | 3 ++ src/runtime/Python.Runtime.csproj | 1 + src/runtime/arrayobject.cs | 4 +- src/runtime/converter.cs | 78 ++++++++++++++----------------- src/runtime/pylong.cs | 2 +- src/runtime/runtime.cs | 39 +++++++--------- src/tests/test_conversion.py | 2 +- src/tests/test_method.py | 3 ++ 8 files changed, 63 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a804e8c..147a716b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ details about the cause of the failure to the regular method return value (unless they are passed with `ref` or `out` keyword). - BREAKING: Drop support for the long-deprecated CLR.* prefix. - `PyObject` now implements `IEnumerable` in addition to `IEnumerable` +- floating point values passed from Python are no longer silently truncated +when .NET expects an integer [#1342][i1342] ### Fixed @@ -809,3 +811,4 @@ This version improves performance on benchmarks significantly compared to 2.3. [i755]: https://github.com/pythonnet/pythonnet/pull/755 [p534]: https://github.com/pythonnet/pythonnet/pull/534 [i449]: https://github.com/pythonnet/pythonnet/issues/449 +[i1342]: https://github.com/pythonnet/pythonnet/issues/1342 diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index f18cf7a49..4939fae34 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -4,6 +4,7 @@ AnyCPU Python.Runtime Python.Runtime + 9.0 pythonnet https://github.com/pythonnet/pythonnet/blob/master/LICENSE https://github.com/pythonnet/pythonnet diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 262e521a5..282383f86 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -54,7 +54,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) // create single dimensional array if (Runtime.PyInt_Check(op)) { - dimensions[0] = Runtime.PyLong_AsLongLong(op); + dimensions[0] = Runtime.PyLong_AsSignedSize_t(op); if (dimensions[0] == -1 && Exceptions.ErrorOccurred()) { Exceptions.Clear(); @@ -89,7 +89,7 @@ static NewReference CreateMultidimensional(Type elementType, long[] dimensions, return default; } - dimensions[dimIndex] = Runtime.PyLong_AsLongLong(dimObj); + dimensions[dimIndex] = Runtime.PyLong_AsSignedSize_t(dimObj); if (dimensions[dimIndex] == -1 && Exceptions.ErrorOccurred()) { Exceptions.RaiseTypeError("array constructor expects integer dimensions"); diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index e1b689cf3..aa4ed6a80 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -517,7 +517,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int32: { // Python3 always use PyLong API - long num = Runtime.PyLong_AsLongLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -547,7 +547,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -573,7 +573,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -610,7 +610,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -625,7 +625,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int16: { - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -640,18 +640,35 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int64: { - long num = (long)Runtime.PyLong_AsLongLong(value); - if (num == -1 && Exceptions.ErrorOccurred()) + if (Runtime.Is32Bit) { - goto convert_error; + if (!Runtime.PyLong_Check(value)) + { + goto type_error; + } + long num = Runtime.PyExplicitlyConvertToInt64(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = num; + return true; + } + else + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = (long)num; + return true; } - result = num; - return true; } case TypeCode.UInt16: { - long num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -666,43 +683,16 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.UInt32: { - op = value; - if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) - { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - goto convert_error; - } - } - if (Runtime.Is32Bit || Runtime.IsWindows) + nuint num = Runtime.PyLong_AsUnsignedSize_t(value); + if (num == unchecked((nuint)(-1)) && Exceptions.ErrorOccurred()) { - uint num = Runtime.PyLong_AsUnsignedLong32(op); - if (num == uint.MaxValue && Exceptions.ErrorOccurred()) - { - goto convert_error; - } - result = num; + goto convert_error; } - else + if (num > UInt32.MaxValue) { - ulong num = Runtime.PyLong_AsUnsignedLong64(op); - if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) - { - goto convert_error; - } - try - { - result = Convert.ToUInt32(num); - } - catch (OverflowException) - { - // Probably wasn't an overflow in python but was in C# (e.g. if cpython - // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in - // PyLong_AsUnsignedLong) - goto overflow; - } + goto overflow; } + result = (uint)num; return true; } diff --git a/src/runtime/pylong.cs b/src/runtime/pylong.cs index 0e21c7c49..fdfd26aba 100644 --- a/src/runtime/pylong.cs +++ b/src/runtime/pylong.cs @@ -246,7 +246,7 @@ public int ToInt32() /// public long ToInt64() { - return Runtime.PyLong_AsLongLong(obj); + return Runtime.PyExplicitlyConvertToInt64(obj); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 63467c917..2454c038f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1254,30 +1254,27 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromString(string value, IntPtr end, int radix); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsUnsignedLong")] - internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); - + EntryPoint = "PyLong_AsSize_t")] + internal static extern nuint PyLong_AsUnsignedSize_t(IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsUnsignedLong")] - internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); - - internal static object PyLong_AsUnsignedLong(IntPtr value) - { - if (Is32Bit || IsWindows) - return PyLong_AsUnsignedLong32(value); - else - return PyLong_AsUnsignedLong64(value); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern long PyLong_AsLongLong(BorrowedReference value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern long PyLong_AsLongLong(IntPtr value); + EntryPoint = "PyLong_AsSsize_t")] + internal static extern nint PyLong_AsSignedSize_t(IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsSsize_t")] + internal static extern nint PyLong_AsSignedSize_t(BorrowedReference value); + /// + /// This function is a rename of PyLong_AsLongLong, which has a commonly undesired + /// behavior to convert everything (including floats) to integer type, before returning + /// the value as . + /// + /// In most cases you need to check that value is an instance of PyLongObject + /// before using this function using . + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsLongLong")] + internal static extern long PyExplicitlyConvertToInt64(IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern ulong PyLong_AsUnsignedLongLong(IntPtr value); diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 313274647..6b152025d 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -343,7 +343,7 @@ def test_uint32_conversion(): ob.UInt32Field = System.UInt32(0) assert ob.UInt32Field == 0 - with pytest.raises(ValueError): + with pytest.raises(TypeError): ConversionTest().UInt32Field = "spam" with pytest.raises(TypeError): diff --git a/src/tests/test_method.py b/src/tests/test_method.py index 18eb5af8e..a69cc6f14 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -807,6 +807,9 @@ def test_no_object_in_param(): with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") + with pytest.raises(TypeError): + MethodTest.TestOverloadedNoObject(5.5) + def test_object_in_param(): """Test regression introduced by #151 in which Object method overloads From 5fd77b11a8441e703b2e2402a1fe495d1c8f8a00 Mon Sep 17 00:00:00 2001 From: gpetrou <4172445+gpetrou@users.noreply.github.com> Date: Thu, 21 Jan 2021 18:27:07 +0000 Subject: [PATCH 015/151] Add PythonEngine.Interrupt (#1337) Also added GetPythonThreadID --- AUTHORS.md | 5 ++- CHANGELOG.md | 1 + src/embed_tests/TestInterrupt.cs | 63 ++++++++++++++++++++++++++++++++ src/runtime/pythonengine.cs | 24 ++++++++++++ src/runtime/runtime.cs | 6 +++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 src/embed_tests/TestInterrupt.cs diff --git a/AUTHORS.md b/AUTHORS.md index 167fd496c..6cfa216b1 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -66,10 +66,10 @@ - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) -- William Sardar ([@williamsardar])(https://github.com/williamsardar) +- William Sardar ([@williamsardar](https://github.com/williamsardar)) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) -- ([@amos402]https://github.com/amos402) +- ([@amos402](https://github.com/amos402)) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) @@ -82,3 +82,4 @@ - ([@testrunner123](https://github.com/testrunner123)) - ([@DanBarzilian](https://github.com/DanBarzilian)) - ([@alxnull](https://github.com/alxnull)) +- ([@gpetrou](https://github.com/gpetrou)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147a716b5..375071841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). +- Add GetPythonThreadID and Interrupt methods in PythonEngine ### Changed - Drop support for Python 2, 3.4, and 3.5 diff --git a/src/embed_tests/TestInterrupt.cs b/src/embed_tests/TestInterrupt.cs new file mode 100644 index 000000000..e075d1194 --- /dev/null +++ b/src/embed_tests/TestInterrupt.cs @@ -0,0 +1,63 @@ + +using System; +using System.Threading; +using System.Threading.Tasks; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInterrupt + { + private IntPtr _threadState; + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + _threadState = PythonEngine.BeginAllowThreads(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.EndAllowThreads(_threadState); + PythonEngine.Shutdown(); + } + + [Test] + public void InterruptTest() + { + int runSimpleStringReturnValue = int.MinValue; + ulong pythonThreadID = ulong.MinValue; + Task.Factory.StartNew(() => + { + using (Py.GIL()) + { + pythonThreadID = PythonEngine.GetPythonThreadID(); + runSimpleStringReturnValue = PythonEngine.RunSimpleString(@" +import time + +while True: + time.sleep(0.2)"); + } + }); + + Thread.Sleep(200); + + Assert.AreNotEqual(ulong.MinValue, pythonThreadID); + + using (Py.GIL()) + { + int interruptReturnValue = PythonEngine.Interrupt(pythonThreadID); + Assert.AreEqual(1, interruptReturnValue); + } + + Thread.Sleep(300); + + Assert.AreEqual(-1, runSimpleStringReturnValue); + } + } +} diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index df6cf7101..781d345e7 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -567,6 +567,30 @@ public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = nu } } + /// + /// Gets the Python thread ID. + /// + /// The Python thread ID. + public static ulong GetPythonThreadID() + { + dynamic threading = Py.Import("threading"); + return threading.InvokeMethod("get_ident"); + } + + /// + /// Interrupts the execution of a thread. + /// + /// The Python thread ID. + /// The number of thread states modified; this is normally one, but will be zero if the thread id isn’t found. + public static int Interrupt(ulong pythonThreadID) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Runtime.PyThreadState_SetAsyncExcLLP64((uint)pythonThreadID, Exceptions.KeyboardInterrupt); + } + + return Runtime.PyThreadState_SetAsyncExcLP64(pythonThreadID, Exceptions.KeyboardInterrupt); + } /// /// RunString Method. Function has been deprecated and will be removed. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2454c038f..8197f027b 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2143,6 +2143,12 @@ internal static void Py_CLEAR(ref IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + [DllImport(_PythonDll, EntryPoint = "PyThreadState_SetAsyncExc", CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyThreadState_SetAsyncExcLLP64(uint id, IntPtr exc); + + [DllImport(_PythonDll, EntryPoint = "PyThreadState_SetAsyncExc", CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyThreadState_SetAsyncExcLP64(ulong id, IntPtr exc); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_MakePendingCalls(); From 7e7cbca1a49409fd46c006412f9951e47a24c77d Mon Sep 17 00:00:00 2001 From: Tom Minka <8955276+tminka@users.noreply.github.com> Date: Thu, 21 Jan 2021 21:07:50 +0000 Subject: [PATCH 016/151] Disable implicit conversion from PyFloat to uint64 (#1362) --- src/runtime/converter.cs | 13 ++----------- src/tests/test_conversion.py | 5 ++++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index aa4ed6a80..62e091d31 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -698,19 +698,10 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.UInt64: { - op = value; - if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) - { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - goto convert_error; - } - } - ulong num = Runtime.PyLong_AsUnsignedLongLong(op); + ulong num = Runtime.PyLong_AsUnsignedLongLong(value); if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) { - goto overflow; + goto convert_error; } result = num; return true; diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 6b152025d..3b290b947 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -382,7 +382,10 @@ def test_uint64_conversion(): ob.UInt64Field = System.UInt64(0) assert ob.UInt64Field == 0 - with pytest.raises(ValueError): + with pytest.raises(TypeError): + ConversionTest().UInt64Field = 0.5 + + with pytest.raises(TypeError): ConversionTest().UInt64Field = "spam" with pytest.raises(TypeError): From 32fdc9c8e393b5d724c5848230a1cf7f6e32d4a3 Mon Sep 17 00:00:00 2001 From: Tom Minka <8955276+tminka@users.noreply.github.com> Date: Thu, 21 Jan 2021 21:09:02 +0000 Subject: [PATCH 017/151] Disable implicit conversion from PyFloat to array index (#1363) --- src/runtime/arrayobject.cs | 23 +++++++++++++++++++++++ src/tests/test_array.py | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 282383f86..5c97c6dbf 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -159,6 +159,10 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (rank == 1) { + if (!Runtime.PyInt_Check(idx)) + { + return RaiseIndexMustBeIntegerError(idx); + } index = Runtime.PyInt_AsLong(idx); if (Exceptions.ErrorOccurred()) @@ -199,6 +203,10 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, for (var i = 0; i < count; i++) { IntPtr op = Runtime.PyTuple_GetItem(idx, i); + if (!Runtime.PyInt_Check(op)) + { + return RaiseIndexMustBeIntegerError(op); + } index = Runtime.PyInt_AsLong(op); if (Exceptions.ErrorOccurred()) @@ -253,6 +261,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (rank == 1) { + if (!Runtime.PyInt_Check(idx)) + { + RaiseIndexMustBeIntegerError(idx); + return -1; + } index = Runtime.PyInt_AsLong(idx); if (Exceptions.ErrorOccurred()) @@ -291,6 +304,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, for (var i = 0; i < count; i++) { IntPtr op = Runtime.PyTuple_GetItem(idx, i); + if (!Runtime.PyInt_Check(op)) + { + RaiseIndexMustBeIntegerError(op); + return -1; + } index = Runtime.PyInt_AsLong(op); if (Exceptions.ErrorOccurred()) @@ -320,6 +338,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, return 0; } + private static IntPtr RaiseIndexMustBeIntegerError(IntPtr idx) + { + string tpName = Runtime.PyObject_GetTypeName(idx); + return Exceptions.RaiseTypeError($"array index has type {tpName}, expected an integer"); + } /// /// Implements __contains__ for array types. diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 9ab044b29..232c89ac7 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -836,8 +836,15 @@ def test_typed_array(): with pytest.raises(TypeError): ob = Test.TypedArrayTest() - ob.items["wrong"] = "wrong" + ob.items["wrong"] = Spam("0") + + with pytest.raises(TypeError): + ob = Test.TypedArrayTest() + _ = ob.items[0.5] + with pytest.raises(TypeError): + ob = Test.TypedArrayTest() + ob.items[0.5] = Spam("0") def test_multi_dimensional_array(): """Test multi-dimensional arrays.""" @@ -901,8 +908,23 @@ def test_multi_dimensional_array(): with pytest.raises(TypeError): ob = Test.MultiDimensionalArrayTest() - ob[0, 0] = "wrong" + ob.items[0, 0] = "wrong" + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob["0", 0] = 0 + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob.items["0", 0] = 0 + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + _ = ob.items[0.5, 0] + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob.items[0.5, 0] = 0 def test_array_iteration(): """Test array iteration.""" @@ -1189,6 +1211,15 @@ def test_create_array_from_shape(): with pytest.raises(ValueError): Array[int](-1) + with pytest.raises(TypeError): + Array[int]('1') + + with pytest.raises(ValueError): + Array[int](-1, -1) + + with pytest.raises(TypeError): + Array[int]('1', '1') + def test_special_array_creation(): """Test using the Array[] syntax for creating arrays.""" from Python.Test import ISayHello1, InterfaceTest, ShortEnum From 92932fd2a615e7ddcf16c4e4441a4840050b6bc9 Mon Sep 17 00:00:00 2001 From: Tom Minka <8955276+tminka@users.noreply.github.com> Date: Wed, 27 Jan 2021 20:12:25 +0000 Subject: [PATCH 018/151] Better error messages for method argument mismatch and others (#1361) Better error messages for method argument mismatch, Python to managed value conversion, and other problems. - Converter.ToManaged obeys setError consistently - PyObject_Hash and tp_hash return nint - MakeGenericType and MakeGenericMethod have try/catch - RaiseTypeError sets __cause__ instead of changing the message string - PythonException.Normalize throws if an error is set - AggregateExceptions print better from Python - For an overloaded method, MethodBinder.Bind collects all binding errors with PythonExceptions as the inner.inner exceptions. - ExceptionClassObject.tp_str calls Exception.ToString() instead of Exception.Message (for justification see https://stackoverflow.com/questions/2176707/exception-message-vs-exception-tostring ) - Binding failures always have AggregateException as the cause, - Disabled test_method_parameters_change --- CHANGELOG.md | 2 + src/clrmodule/ClrModule.cs | 4 +- src/domain_tests/test_domain_reload.py | 1 + src/embed_tests/TestPythonException.cs | 10 ++ src/runtime/classbase.cs | 15 ++- src/runtime/converter.cs | 73 +++++++++--- src/runtime/eventbinding.cs | 22 ++-- src/runtime/eventobject.cs | 11 +- src/runtime/exceptions.cs | 46 ++++++-- src/runtime/methodbinder.cs | 149 ++++++++++++++++++++----- src/runtime/methodbinding.cs | 22 ++-- src/runtime/pythonexception.cs | 21 ++++ src/runtime/runtime.cs | 21 +++- src/runtime/typemanager.cs | 4 +- src/testing/Python.Test.csproj | 1 + src/testing/generictest.cs | 11 ++ src/testing/indexertest.cs | 12 +- src/testing/methodtest.cs | 21 ++-- src/tests/test_generic.py | 34 +++++- src/tests/test_indexer.py | 18 +++ src/tests/test_method.py | 28 +++-- src/tests/test_subclass.py | 13 +++ 22 files changed, 429 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 375071841..a332f057d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ details about the cause of the failure - `PyObject` now implements `IEnumerable` in addition to `IEnumerable` - floating point values passed from Python are no longer silently truncated when .NET expects an integer [#1342][i1342] +- More specific error messages for method argument mismatch ### Fixed @@ -49,6 +50,7 @@ when .NET expects an integer [#1342][i1342] - Fixed objects returned by enumerating `PyObject` being disposed too soon - Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException - `import` may now raise errors with more detail than "No module named X" +- Providing an invalid type parameter to a generic type or method produces a helpful Python error ### Removed diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 7b0387d46..ab0f6da9f 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -62,9 +62,9 @@ public static IntPtr PyInit_clr() pythonRuntime = Assembly.Load(pythonRuntimeName); DebugPrint("Success loading 'Python.Runtime' using standard binding rules."); } - catch (IOException) + catch (IOException ex) { - DebugPrint("'Python.Runtime' not found using standard binding rules."); + DebugPrint($"'Python.Runtime' not found using standard binding rules: {ex}"); try { // If the above fails for any reason, we fallback to attempting to load "Python.Runtime.dll" diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index e24eb6976..fa6f42b9e 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -57,6 +57,7 @@ def test_property_visibility_change(): def test_class_visibility_change(): _run_test("class_visibility_change") +@pytest.mark.skip(reason='FIXME: Domain reload fails when Python points to a .NET object which points back to Python objects') @pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_parameters_change(): _run_test("method_parameters_change") diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 3ab0f8106..e51da4d4f 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -97,6 +97,7 @@ public void TestPythonExceptionFormatNormalized() try { PythonEngine.Exec("a=b\n"); + Assert.Fail("Exception should have been raised"); } catch (PythonException ex) { @@ -135,5 +136,14 @@ def __init__(self, val): } } } + + [Test] + public void TestPythonException_Normalize_ThrowsWhenErrorSet() + { + Exceptions.SetError(Exceptions.TypeError, "Error!"); + var pythonException = new PythonException(); + Exceptions.SetError(Exceptions.TypeError, "Another error"); + Assert.Throws(() => pythonException.Normalize()); + } } } diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 872501267..8b96a96da 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -66,7 +66,16 @@ public virtual IntPtr type_subscript(IntPtr idx) if (target != null) { - Type t = target.MakeGenericType(types); + Type t; + try + { + // MakeGenericType can throw ArgumentException + t = target.MakeGenericType(types); + } + catch (ArgumentException e) + { + return Exceptions.RaiseTypeError(e.Message); + } ManagedType c = ClassManager.GetClass(t); Runtime.XIncref(c.pyHandle); return c.pyHandle; @@ -263,14 +272,14 @@ public static IntPtr tp_iter(IntPtr ob) /// /// Standard __hash__ implementation for instances of reflected types. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var co = GetManagedObject(ob) as CLRObject; if (co == null) { return Exceptions.RaiseTypeError("unhashable type"); } - return new IntPtr(co.inst.GetHashCode()); + return co.inst.GetHashCode(); } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 62e091d31..0f263c721 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -303,6 +303,11 @@ internal static IntPtr ToPythonImplicit(object value) /// Return a managed object for the given Python object, taking funny /// byref types into account. /// + /// A Python object + /// The desired managed type + /// Receives the managed object + /// If true, call Exceptions.SetError with the reason for failure. + /// True on success internal static bool ToManaged(IntPtr value, Type type, out object result, bool setError) { @@ -341,7 +346,10 @@ internal static bool ToManagedValue(IntPtr value, Type obType, result = tmp; return true; } - Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); + } return false; } if (mt is ClassBase) @@ -376,6 +384,15 @@ internal static bool ToManagedValue(IntPtr value, Type obType, obType = obType.GetGenericArguments()[0]; } + if (obType.ContainsGenericParameters) + { + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, $"Cannot create an instance of the open generic type {obType}"); + } + return false; + } + if (obType.IsArray) { return ToArray(value, obType, out result, setError); @@ -777,7 +794,7 @@ private static void SetConversionError(IntPtr value, Type target) IntPtr ob = Runtime.PyObject_Repr(value); string src = Runtime.GetManagedString(ob); Runtime.XDecref(ob); - Exceptions.SetError(Exceptions.TypeError, $"Cannot convert {src} to {target}"); + Exceptions.RaiseTypeError($"Cannot convert {src} to {target}"); } @@ -791,32 +808,58 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s Type elementType = obType.GetElementType(); result = null; - bool IsSeqObj = Runtime.PySequence_Check(value); - var len = IsSeqObj ? Runtime.PySequence_Size(value) : -1; - IntPtr IterObject = Runtime.PyObject_GetIter(value); - - if(IterObject==IntPtr.Zero) { + if (IterObject == IntPtr.Zero) + { if (setError) { SetConversionError(value, obType); } + else + { + // PyObject_GetIter will have set an error + Exceptions.Clear(); + } return false; } - Array items; + IList list; + try + { + // MakeGenericType can throw because elementType may not be a valid generic argument even though elementType[] is a valid array type. + // For example, if elementType is a pointer type. + // See https://docs.microsoft.com/en-us/dotnet/api/system.type.makegenerictype#System_Type_MakeGenericType_System_Type + var constructedListType = typeof(List<>).MakeGenericType(elementType); + bool IsSeqObj = Runtime.PySequence_Check(value); + if (IsSeqObj) + { + var len = Runtime.PySequence_Size(value); + list = (IList)Activator.CreateInstance(constructedListType, new Object[] { (int)len }); + } + else + { + // CreateInstance can throw even if MakeGenericType succeeded. + // See https://docs.microsoft.com/en-us/dotnet/api/system.activator.createinstance#System_Activator_CreateInstance_System_Type_ + list = (IList)Activator.CreateInstance(constructedListType); + } + } + catch (Exception e) + { + if (setError) + { + Exceptions.SetError(e); + SetConversionError(value, obType); + } + return false; + } - var listType = typeof(List<>); - var constructedListType = listType.MakeGenericType(elementType); - IList list = IsSeqObj ? (IList) Activator.CreateInstance(constructedListType, new Object[] {(int) len}) : - (IList) Activator.CreateInstance(constructedListType); IntPtr item; while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) { - object obj = null; + object obj; - if (!Converter.ToManaged(item, elementType, out obj, true)) + if (!Converter.ToManaged(item, elementType, out obj, setError)) { Runtime.XDecref(item); return false; @@ -827,7 +870,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s } Runtime.XDecref(IterObject); - items = Array.CreateInstance(elementType, list.Count); + Array items = Array.CreateInstance(elementType, list.Count); list.CopyTo(items, 0); result = items; diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 581095185..3f5b7b007 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -68,35 +68,27 @@ public static IntPtr nb_inplace_subtract(IntPtr ob, IntPtr arg) /// /// EventBinding __hash__ implementation. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); - long x = 0; - long y = 0; + nint x = 0; if (self.target != IntPtr.Zero) { - x = Runtime.PyObject_Hash(self.target).ToInt64(); + x = Runtime.PyObject_Hash(self.target); if (x == -1) { - return new IntPtr(-1); + return x; } } - y = Runtime.PyObject_Hash(self.e.pyHandle).ToInt64(); + nint y = Runtime.PyObject_Hash(self.e.pyHandle); if (y == -1) { - return new IntPtr(-1); + return y; } - x ^= y; - - if (x == -1) - { - x = -1; - } - - return new IntPtr(x); + return x ^ y; } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 0f2796a14..4dc785ddd 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -72,6 +72,12 @@ internal bool AddEventHandler(IntPtr target, IntPtr handler) /// internal bool RemoveEventHandler(IntPtr target, IntPtr handler) { + if (reg == null) + { + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + object obj = null; if (target != IntPtr.Zero) { @@ -79,10 +85,9 @@ internal bool RemoveEventHandler(IntPtr target, IntPtr handler) obj = co.inst; } - IntPtr hash = Runtime.PyObject_Hash(handler); - if (Exceptions.ErrorOccurred() || reg == null) + nint hash = Runtime.PyObject_Hash(handler); + if (hash == -1 && Exceptions.ErrorOccurred()) { - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); return false; } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index ab28905d2..9091fd071 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -1,6 +1,7 @@ using System; using System.Reflection; using System.Runtime.InteropServices; +using System.Text; namespace Python.Runtime { @@ -71,14 +72,16 @@ internal static Exception ToException(IntPtr ob) return Exceptions.RaiseTypeError("invalid object"); } - string message = string.Empty; - if (e.Message != string.Empty) + string message = e.ToString(); + string fullTypeName = e.GetType().FullName; + string prefix = fullTypeName + ": "; + if (message.StartsWith(prefix)) { - message = e.Message; + message = message.Substring(prefix.Length); } - if (!string.IsNullOrEmpty(e.StackTrace)) + else if (message.StartsWith(fullTypeName)) { - message = message + "\n" + e.StackTrace; + message = message.Substring(fullTypeName.Length); } return Runtime.PyUnicode_FromString(message); } @@ -181,6 +184,7 @@ internal static void SetArgsAndCause(IntPtr ob) if (e.InnerException != null) { + // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. IntPtr cause = CLRObject.GetInstHandle(e.InnerException); Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); } @@ -290,6 +294,20 @@ public static void SetError(Exception e) Runtime.XDecref(op); } + /// + /// When called after SetError, sets the cause of the error. + /// + /// The cause of the current error + public static void SetCause(PythonException cause) + { + var currentException = new PythonException(); + currentException.Normalize(); + cause.Normalize(); + Runtime.XIncref(cause.PyValue); + Runtime.PyException_SetCause(currentException.PyValue, cause.PyValue); + currentException.Restore(); + } + /// /// ErrorOccurred Method /// @@ -368,17 +386,31 @@ public static void deprecation(string message) // Internal helper methods for common error handling scenarios. //==================================================================== + /// + /// Raises a TypeError exception and attaches any existing exception as its cause. + /// + /// The exception message + /// IntPtr.Zero internal static IntPtr RaiseTypeError(string message) { + PythonException previousException = null; + if (ErrorOccurred()) + { + previousException = new PythonException(); + } Exceptions.SetError(Exceptions.TypeError, message); + if (previousException != null) + { + SetCause(previousException); + } return IntPtr.Zero; } // 2010-11-16: Arranged in python (2.6 & 2.7) source header file order /* Predefined exceptions are - puplic static variables on the Exceptions class filled in from + public static variables on the Exceptions class filled in from the python class using reflection in Initialize() looked up by - name, not posistion. */ + name, not position. */ public static IntPtr BaseException; public static IntPtr Exception; public static IntPtr StopIteration; diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5de0ecc00..3f879d3c4 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -17,6 +17,9 @@ namespace Python.Runtime [Serializable] internal class MethodBinder { + /// + /// The overloads of this method + /// public List list; [NonSerialized] @@ -83,6 +86,7 @@ internal static MethodInfo MatchSignature(MethodInfo[] mi, Type[] tp) /// /// Given a sequence of MethodInfo and a sequence of type parameters, /// return the MethodInfo that represents the matching closed generic. + /// If unsuccessful, returns null and may set a Python error. /// internal static MethodInfo MatchParameters(MethodInfo[] mi, Type[] tp) { @@ -102,7 +106,18 @@ internal static MethodInfo MatchParameters(MethodInfo[] mi, Type[] tp) { continue; } - return t.MakeGenericMethod(tp); + try + { + // MakeGenericMethod can throw ArgumentException if the type parameters do not obey the constraints. + MethodInfo method = t.MakeGenericMethod(tp); + Exceptions.Clear(); + return method; + } + catch (ArgumentException e) + { + Exceptions.SetError(e); + // The error will remain set until cleared by a successful match. + } } return null; } @@ -277,20 +292,37 @@ internal static int ArgPrecedence(Type t) /// /// Bind the given Python instance and arguments to a particular method - /// overload and return a structure that contains the converted Python + /// overload in and return a structure that contains the converted Python /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw) { return Bind(inst, args, kw, null, null); } + /// + /// Bind the given Python instance and arguments to a particular method + /// overload in and return a structure that contains the converted Python + /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. + /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// If not null, only bind to that method. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { return Bind(inst, args, kw, info, null); } - private readonly struct MatchedMethod { + private readonly struct MatchedMethod + { public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int outs, MethodBase mb) { KwargsMatched = kwargsMatched; @@ -307,9 +339,33 @@ public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int public MethodBase Method { get; } } + private readonly struct MismatchedMethod + { + public MismatchedMethod(PythonException exception, MethodBase mb) + { + Exception = exception; + Method = mb; + } + + public PythonException Exception { get; } + public MethodBase Method { get; } + } + + /// + /// Bind the given Python instance and arguments to a particular method + /// overload in and return a structure that contains the converted Python + /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. + /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// If not null, only bind to that method. + /// If not null, additionally attempt to bind to the generic methods in this array by inferring generic type parameters. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { - // loop to find match, return invoker w/ or /wo error + // loop to find match, return invoker w/ or w/o error MethodBase[] _methods = null; var kwargDict = new Dictionary(); @@ -340,6 +396,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } var argMatchedMethods = new List(_methods.Length); + var mismatchedMethods = new List(); // TODO: Clean up foreach (MethodBase mi in _methods) @@ -375,12 +432,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // We need to take the first CLR argument. pi = pi.Take(1).ToArray(); } - var outs = 0; + int outs; var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, needsResolution: _methods.Length > 1, // If there's more than one possible match. outs: out outs); if (margs == null) { + mismatchedMethods.Add(new MismatchedMethod(new PythonException(), mi)); + Exceptions.Clear(); continue; } if (isOperator) @@ -403,7 +462,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } margs = margsTemp; } - else { break; } + else continue; } } @@ -434,6 +493,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // Best effort for determining method to match on gives multiple possible // matches and we need at least one default argument - bail from this point + StringBuilder stringBuilder = new StringBuilder("Not enough arguments provided to disambiguate the method. Found:"); + foreach (var matchedMethod in argMatchedMethods) + { + stringBuilder.AppendLine(); + stringBuilder.Append(matchedMethod.Method.ToString()); + } + Exceptions.SetError(Exceptions.TypeError, stringBuilder.ToString()); return null; } @@ -463,6 +529,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // XXX maybe better to do this before all the other rigmarole. if (co == null) { + Exceptions.SetError(Exceptions.TypeError, "Invoked a non-static method with an invalid instance"); return null; } target = co.inst; @@ -470,19 +537,32 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return new Binding(mi, target, margs, outs); } - // We weren't able to find a matching method but at least one - // is a generic method and info is null. That happens when a generic - // method was not called using the [] syntax. Let's introspect the - // type of the arguments and use it to construct the correct method. - if (isGeneric && info == null && methodinfo != null) + else if (isGeneric && info == null && methodinfo != null) { + // We weren't able to find a matching method but at least one + // is a generic method and info is null. That happens when a generic + // method was not called using the [] syntax. Let's introspect the + // type of the arguments and use it to construct the correct method. Type[] types = Runtime.PythonArgsToTypeArray(args, true); MethodInfo mi = MatchParameters(methodinfo, types); - return Bind(inst, args, kw, mi, null); + if (mi != null) + { + return Bind(inst, args, kw, mi, null); + } + } + if (mismatchedMethods.Count > 0) + { + var aggregateException = GetAggregateException(mismatchedMethods); + Exceptions.SetError(aggregateException); } return null; } + static AggregateException GetAggregateException(IEnumerable mismatchedMethods) + { + return new AggregateException(mismatchedMethods.Select(m => new ArgumentException($"{m.Exception.Message} in method {m.Method}", m.Exception))); + } + static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out bool isNewReference) { isNewReference = false; @@ -517,6 +597,7 @@ static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out /// /// Attempts to convert Python positional argument tuple and keyword argument table /// into an array of managed objects, that can be passed to a method. + /// If unsuccessful, returns null and may set a Python error. /// /// Information about expected parameters /// true, if the last parameter is a params array. @@ -526,7 +607,7 @@ static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out /// A list of default values for omitted parameters /// true, if overloading resolution is required /// Returns number of output parameters - /// An array of .NET arguments, that can be passed to a method. + /// If successful, an array of .NET arguments that can be passed to the method. Otherwise null. static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, IntPtr args, int pyArgCount, Dictionary kwargDict, @@ -596,13 +677,14 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, /// /// Try to convert a Python argument object to a managed CLR type. + /// If unsuccessful, may set a Python error. /// - /// Pointer to the object at a particular parameter. + /// Pointer to the Python argument object. /// That parameter's managed type. - /// There are multiple overloading methods that need resolution. + /// If true, there are multiple overloading methods that need resolution. /// Converted argument. /// Whether the CLR type is passed by reference. - /// + /// true on success static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, out object arg, out bool isOut) { @@ -614,9 +696,8 @@ static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResoluti return false; } - if (!Converter.ToManaged(op, clrtype, out arg, false)) + if (!Converter.ToManaged(op, clrtype, out arg, true)) { - Exceptions.Clear(); return false; } @@ -624,6 +705,13 @@ static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResoluti return true; } + /// + /// Determine the managed type that a Python argument object needs to be converted into. + /// + /// The parameter's managed type. + /// Pointer to the Python argument object. + /// If true, there are multiple overloading methods that need resolution. + /// null if conversion is not possible static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool needsResolution) { // this logic below handles cases when multiple overloading methods @@ -635,7 +723,6 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool { // HACK: each overload should be weighted in some way instead pyoptype = Runtime.PyObject_Type(argument); - Exceptions.Clear(); if (pyoptype != IntPtr.Zero) { clrtype = Converter.GetTypeByAlias(pyoptype); @@ -645,12 +732,11 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool if (clrtype != null) { - var typematch = false; if ((parameterType != typeof(object)) && (parameterType != clrtype)) { IntPtr pytype = Converter.GetPythonTypeByAlias(parameterType); pyoptype = Runtime.PyObject_Type(argument); - Exceptions.Clear(); + var typematch = false; if (pyoptype != IntPtr.Zero) { if (pytype != pyoptype) @@ -666,13 +752,17 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool if (!typematch) { // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(parameterType); - TypeCode paramtypecode = Type.GetTypeCode(clrtype); - if (argtypecode == paramtypecode) + TypeCode parameterTypeCode = Type.GetTypeCode(parameterType); + TypeCode clrTypeCode = Type.GetTypeCode(clrtype); + if (parameterTypeCode == clrTypeCode) { typematch = true; clrtype = parameterType; } + else + { + Exceptions.RaiseTypeError($"Expected {parameterTypeCode}, got {clrTypeCode}"); + } } Runtime.XDecref(pyoptype); if (!typematch) @@ -682,7 +772,6 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool } else { - typematch = true; clrtype = parameterType; } } @@ -815,9 +904,9 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i var msg = new StringBuilder("The underlying C# method(s) have been deleted"); if (list.Count > 0 && list[0].Name != null) { - msg.Append($": {list[0].ToString()}"); + msg.Append($": {list[0]}"); } - return Exceptions.RaiseTypeError(msg.ToString());; + return Exceptions.RaiseTypeError(msg.ToString()); } Binding binding = Bind(inst, args, kw, info, methodinfo); @@ -831,10 +920,14 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i { value.Append($" for {methodinfo[0].Name}"); } + else if (list.Count > 0 && list[0].Valid) + { + value.Append($" for {list[0].Value.Name}"); + } value.Append(": "); AppendArgumentTypes(to: value, args); - Exceptions.SetError(Exceptions.TypeError, value.ToString()); + Exceptions.RaiseTypeError(value.ToString()); return IntPtr.Zero; } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 46b62807d..f33015ba4 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -201,35 +201,27 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) /// /// MethodBinding __hash__ implementation. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - long x = 0; - long y = 0; + nint x = 0; if (self.target != IntPtr.Zero) { - x = Runtime.PyObject_Hash(self.target).ToInt64(); + x = Runtime.PyObject_Hash(self.target); if (x == -1) { - return new IntPtr(-1); + return x; } } - y = Runtime.PyObject_Hash(self.m.pyHandle).ToInt64(); + nint y = Runtime.PyObject_Hash(self.m.pyHandle); if (y == -1) { - return new IntPtr(-1); + return y; } - x ^= y; - - if (x == -1) - { - x = -1; - } - - return new IntPtr(x); + return x ^ y; } /// diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index eff33d699..ad4d79960 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -36,6 +36,7 @@ public PythonException() _pythonTypeName = type; + // TODO: If pyValue has a __cause__ attribute, then we could set this.InnerException to the equivalent managed exception. Runtime.XIncref(_pyValue); using (var pyValue = new PyObject(_pyValue)) { @@ -148,6 +149,26 @@ public string PythonTypeName get { return _pythonTypeName; } } + /// + /// Replaces PyValue with an instance of PyType, if PyValue is not already an instance of PyType. + /// Often PyValue is a string and this method will replace it with a proper exception object. + /// Must not be called when an error is set. + /// + public void Normalize() + { + IntPtr gs = PythonEngine.AcquireLock(); + try + { + if (Exceptions.ErrorOccurred()) throw new InvalidOperationException("Cannot normalize when an error is set"); + // If an error is set and this PythonException is unnormalized, the error will be cleared and the PythonException will be replaced by a different error. + Runtime.PyErr_NormalizeException(ref _pyType, ref _pyValue, ref _pyTB); + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + /// /// Formats this PythonException object into a message as would be printed /// out via the Python console. See traceback.format_exception diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 8197f027b..c754b80b5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1114,7 +1114,7 @@ internal static long PyObject_Size(IntPtr pointer) private static extern IntPtr _PyObject_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Hash(IntPtr op); + internal static extern nint PyObject_Hash(IntPtr op); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Repr(IntPtr pointer); @@ -1947,8 +1947,11 @@ internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) return (type == ofType) || PyType_IsSubtype(type, ofType); } + /// + /// Generic handler for the tp_new slot of a type object. Create a new instance using the type’s tp_alloc slot. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); + internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kwds); internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) { @@ -2034,6 +2037,14 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val); + /// + /// Under certain circumstances, the values returned by PyErr_Fetch() below can be “unnormalized”, + /// meaning that *exc is a class object but *val is not an instance of the same class. + /// This function can be used to instantiate the class in that case. + /// If the values are already normalized, nothing happens. + /// The delayed normalization is implemented to improve performance. + /// Must not be called when an error is set. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); @@ -2052,6 +2063,12 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + /// + /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void PyException_SetCause(IntPtr ex, IntPtr cause); + //==================================================================== // Cell API //==================================================================== diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 973a5aea2..3e9e44a46 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -332,7 +332,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { if (Exceptions.ErrorOccurred()) return IntPtr.Zero; } - else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, false)) + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, true)) { return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); } @@ -344,7 +344,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { if (Exceptions.ErrorOccurred()) return IntPtr.Zero; } - else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, false)) + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, true)) { return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); } diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index e0c0ca8b1..e6e11c1da 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,6 +1,7 @@ netstandard2.0 + true diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index 1e9c71ac6..238435811 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -35,6 +35,9 @@ public DerivedFromOpenGeneric(int arg1, V arg2, W arg3) : base(arg1, arg2) } } + public class GenericTypeWithConstraint + where T: struct + { } public class GenericNameTest1 { @@ -125,4 +128,12 @@ public static string Overloaded(int arg1, int arg2, string arg3) return arg3; } } + + public class GenericArrayConversionTest + { + public static T[] EchoRange(T[] items) + { + return items; + } + } } diff --git a/src/testing/indexertest.cs b/src/testing/indexertest.cs index 78bb99a7c..08e6ad053 100644 --- a/src/testing/indexertest.cs +++ b/src/testing/indexertest.cs @@ -427,7 +427,8 @@ public interface IIndexer public interface IInheritedIndexer : IIndexer { } - public class InterfaceInheritedIndexerTest :IndexerBase, IInheritedIndexer { + public class InterfaceInheritedIndexerTest : IndexerBase, IInheritedIndexer + { private System.Collections.Generic.IDictionary d = new System.Collections.Generic.Dictionary(); public string this[int index] @@ -436,4 +437,13 @@ public string this[int index] set { t[index] = value; } } } + + public class PublicInheritedOverloadedIndexer : PublicIndexerTest + { + public string this[string index] + { + get { return GetValue(index); } + set { t[index] = value; } + } + } } diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index f5d694488..abdc5f4d6 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -85,7 +85,7 @@ public Type[] TestNullArrayConversion(Type[] v) public static string[] TestStringParamsArg(params string[] args) { - return args.Concat(new []{"tail"}).ToArray(); + return args.Concat(new[] { "tail" }).ToArray(); } public static object[] TestObjectParamsArg(params object[] args) @@ -654,32 +654,32 @@ public static string Casesensitive() return "Casesensitive"; } - public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + public static string DefaultParams(int a = 0, int b = 0, int c = 0, int d = 0) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + public static string OptionalParams([Optional] int a, [Optional] int b, [Optional] int c, [Optional] int d) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static bool OptionalParams_TestMissing([Optional]object a) + public static bool OptionalParams_TestMissing([Optional] object a) { return a == Type.Missing; } - public static bool OptionalParams_TestReferenceType([Optional]string a) + public static bool OptionalParams_TestReferenceType([Optional] string a) { return a == null; } - public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + public static string OptionalAndDefaultParams([Optional] int a, [Optional] int b, int c = 0, int d = 0) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + public static string OptionalAndDefaultParams2([Optional] int a, [Optional] int b, [Optional, DefaultParameterValue(1)] int c, int d = 2) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } @@ -717,6 +717,13 @@ public static string ParamsArrayOverloaded(int i, params int[] paramsArray) public static void EncodingTestÅngström() { } + + // This method can never be invoked from Python, but we want to test that any attempt fails gracefully instead of crashing. + unsafe + public static void PointerArray(int*[] array) + { + + } } diff --git a/src/tests/test_generic.py b/src/tests/test_generic.py index c865ab32f..248303179 100644 --- a/src/tests/test_generic.py +++ b/src/tests/test_generic.py @@ -298,9 +298,8 @@ def test_generic_method_type_handling(): from Python.Test import InterfaceTest, ISayHello1, ShortEnum import System - # FIXME: The value doesn't fit into Int64 and PythonNet doesn't - # recognize it as UInt64 for unknown reasons. - # assert_generic_method_by_type(System.UInt64, 18446744073709551615L) + # FIXME: Fails because Converter.GetTypeByAlias returns int32 for any integer, including ulong + # assert_generic_method_by_type(System.UInt64, 18446744073709551615) assert_generic_method_by_type(System.Boolean, True) assert_generic_method_by_type(bool, True) assert_generic_method_by_type(System.Byte, 255) @@ -750,3 +749,32 @@ def test_missing_generic_type(): from System.Collections import IList with pytest.raises(TypeError): IList[bool] + +def test_invalid_generic_type_parameter(): + from Python.Test import GenericTypeWithConstraint + with pytest.raises(TypeError): + GenericTypeWithConstraint[System.Object] + +def test_invalid_generic_method_type_parameter(): + from Python.Test import ConversionTest + with pytest.raises(TypeError): + ConversionTest.Echo[System.Object] + +def test_generic_list_array_conversion(): + """Test conversion of lists to generic array arguments.""" + from Python.Test import GenericArrayConversionTest + from Python.Test import Spam + + items = [] + for i in range(10): + items.append(Spam(str(i))) + + result = GenericArrayConversionTest.EchoRange[Spam](items) + assert result[0].__class__ == Spam + assert len(result) == 10 + + # Currently raises an exception because the correct generic type parameter is not inferred + with pytest.raises(TypeError): + result = GenericArrayConversionTest.EchoRange(items) + assert result[0].__class__ == Spam + assert len(result) == 10 diff --git a/src/tests/test_indexer.py b/src/tests/test_indexer.py index b8a33fb46..7992f76b0 100644 --- a/src/tests/test_indexer.py +++ b/src/tests/test_indexer.py @@ -646,3 +646,21 @@ def test_inherited_indexer_interface(): ifc = IInheritedIndexer(impl) ifc[0] = "zero" assert ifc[0] == "zero" + +def test_public_inherited_overloaded_indexer(): + """Test public indexers.""" + ob = Test.PublicInheritedOverloadedIndexer() + + ob[0] = "zero" + assert ob[0] == "zero" + + ob[1] = "one" + assert ob[1] == "one" + + assert ob[10] is None + + ob["spam"] = "spam" + assert ob["spam"] == "spam" + + with pytest.raises(TypeError): + ob[[]] diff --git a/src/tests/test_method.py b/src/tests/test_method.py index a69cc6f14..2826ad467 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -810,6 +810,9 @@ def test_no_object_in_param(): with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject(5.5) + # Ensure that the top-level error is TypeError even if the inner error is an OverflowError + with pytest.raises(TypeError): + MethodTest.TestOverloadedNoObject(2147483648) def test_object_in_param(): """Test regression introduced by #151 in which Object method overloads @@ -889,9 +892,16 @@ def test_object_in_multiparam(): def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" - with pytest.raises(TypeError): + with pytest.raises(TypeError) as excinfo: MethodTest.TestOverloadedObjectThree("foo", "bar") + e = excinfo.value + c = e.__cause__ + assert c.GetType().FullName == 'System.AggregateException' + assert len(c.InnerExceptions) == 2 + message = 'One or more errors occurred.' + s = str(c) + assert s[0:len(message)] == message def test_case_sensitive(): """Test that case-sensitivity is respected. GH#81""" @@ -1216,13 +1226,17 @@ def test_params_array_overload(): res = MethodTest.ParamsArrayOverloaded(1, 2, 3, i=1) assert res == "with params-array" - # These two cases are still incorrectly failing: - - # res = MethodTest.ParamsArrayOverloaded(1, 2, i=1) - # assert res == "with params-array" +@pytest.mark.skip(reason="FIXME: incorrectly failing") +def test_params_array_overloaded_failing(): + res = MethodTest.ParamsArrayOverloaded(1, 2, i=1) + assert res == "with params-array" - # res = MethodTest.ParamsArrayOverloaded(paramsArray=[], i=1) - # assert res == "with params-array" + res = MethodTest.ParamsArrayOverloaded(paramsArray=[], i=1) + assert res == "with params-array" def test_method_encoding(): MethodTest.EncodingTestÅngström() + +def test_method_with_pointer_array_argument(): + with pytest.raises(TypeError): + MethodTest.PointerArray([0]) diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py index 968f6a788..4f3180480 100644 --- a/src/tests/test_subclass.py +++ b/src/tests/test_subclass.py @@ -57,6 +57,14 @@ def return_list(self): return DerivedClass +def broken_derived_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class DerivedClass(SubClassTest): + """class that derives from a class deriving from IInterfaceTest""" + __namespace__ = 3 + + return DerivedClass def derived_event_test_class_fixture(subnamespace): """Delay creation of class until test starts.""" @@ -127,6 +135,11 @@ def test_derived_class(): x = FunctionsTest.pass_through(ob) assert id(x) == id(ob) +def test_broken_derived_class(): + """Test python class derived from managed type with invalid namespace""" + with pytest.raises(TypeError): + DerivedClass = broken_derived_class_fixture(test_derived_class.__name__) + ob = DerivedClass() def test_derived_traceback(): """Test python exception traceback in class derived from managed base""" From 063a674041018abd52f09646d4124970cf5722b6 Mon Sep 17 00:00:00 2001 From: Tom Minka <8955276+tminka@users.noreply.github.com> Date: Thu, 28 Jan 2021 02:07:28 +0000 Subject: [PATCH 019/151] Added the ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. BREAKING: MethodBinder omits a void return type when returning a tuple of out parameters. DelegateManager unpacks a tuple of out parameters from Python (reversing the logic in MethodBinder) and sets the out parameters of the delegate. --- CHANGELOG.md | 6 +- src/runtime/converter.cs | 10 +- src/runtime/delegatemanager.cs | 203 +++++++++++++++++++++++++++------ src/runtime/methodbinder.cs | 25 ++-- src/testing/delegatetest.cs | 43 +++++++ src/testing/eventtest.cs | 46 +++++++- src/tests/test_delegate.py | 160 ++++++++++++++++++++++++++ src/tests/test_event.py | 35 ++++++ 8 files changed, 471 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a332f057d..60d516488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine +- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) ### Changed - Drop support for Python 2, 3.4, and 3.5 @@ -32,6 +33,7 @@ details about the cause of the failure - floating point values passed from Python are no longer silently truncated when .NET expects an integer [#1342][i1342] - More specific error messages for method argument mismatch +- BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. ### Fixed @@ -48,8 +50,8 @@ when .NET expects an integer [#1342][i1342] - Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions - Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects - Fixed objects returned by enumerating `PyObject` being disposed too soon -- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException -- `import` may now raise errors with more detail than "No module named X" +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException ([#1325][i1325]) +- `import` may now raise errors with more detail than "No module named X" - Providing an invalid type parameter to a generic type or method produces a helpful Python error ### Removed diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 0f263c721..54124ad34 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -338,9 +338,9 @@ internal static bool ToManagedValue(IntPtr value, Type obType, if (mt != null) { - if (mt is CLRObject) + if (mt is CLRObject co) { - object tmp = ((CLRObject)mt).inst; + object tmp = co.inst; if (obType.IsInstanceOfType(tmp)) { result = tmp; @@ -348,13 +348,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (setError) { - Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); + string typeString = tmp is null ? "null" : tmp.GetType().ToString(); + Exceptions.SetError(Exceptions.TypeError, $"{typeString} value cannot be converted to {obType}"); } return false; } - if (mt is ClassBase) + if (mt is ClassBase cb) { - var cb = (ClassBase)mt; if (!cb.type.Valid) { Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 3e6541c44..0a848904a 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -1,7 +1,9 @@ using System; -using System.Collections; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Reflection.Emit; +using System.Text; namespace Python.Runtime { @@ -11,23 +13,20 @@ namespace Python.Runtime /// internal class DelegateManager { - private Hashtable cache; - private Type basetype; - private Type listtype; - private Type voidtype; - private Type typetype; - private Type ptrtype; - private CodeGenerator codeGenerator; + private readonly Dictionary cache = new Dictionary(); + private readonly Type basetype = typeof(Dispatcher); + private readonly Type arrayType = typeof(object[]); + private readonly Type voidtype = typeof(void); + private readonly Type typetype = typeof(Type); + private readonly Type ptrtype = typeof(IntPtr); + private readonly CodeGenerator codeGenerator = new CodeGenerator(); + private readonly ConstructorInfo arrayCtor; + private readonly MethodInfo dispatch; public DelegateManager() { - basetype = typeof(Dispatcher); - listtype = typeof(ArrayList); - voidtype = typeof(void); - typetype = typeof(Type); - ptrtype = typeof(IntPtr); - cache = new Hashtable(); - codeGenerator = new CodeGenerator(); + arrayCtor = arrayType.GetConstructor(new[] { typeof(int) }); + dispatch = basetype.GetMethod("Dispatch"); } /// @@ -58,10 +57,9 @@ private Type GetDispatcher(Type dtype) // unique signatures rather than delegate types, since multiple // delegate types with the same sig could use the same dispatcher. - object item = cache[dtype]; - if (item != null) + if (cache.TryGetValue(dtype, out Type item)) { - return (Type)item; + return item; } string name = $"__{dtype.FullName}Dispatcher"; @@ -103,34 +101,77 @@ private Type GetDispatcher(Type dtype) MethodBuilder mb = tb.DefineMethod("Invoke", MethodAttributes.Public, method.ReturnType, signature); - ConstructorInfo ctor = listtype.GetConstructor(Type.EmptyTypes); - MethodInfo dispatch = basetype.GetMethod("Dispatch"); - MethodInfo add = listtype.GetMethod("Add"); - il = mb.GetILGenerator(); - il.DeclareLocal(listtype); - il.Emit(OpCodes.Newobj, ctor); + // loc_0 = new object[pi.Length] + il.DeclareLocal(arrayType); + il.Emit(OpCodes.Ldc_I4, pi.Length); + il.Emit(OpCodes.Newobj, arrayCtor); il.Emit(OpCodes.Stloc_0); + bool anyByRef = false; + for (var c = 0; c < signature.Length; c++) { Type t = signature[c]; il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + if (t.IsByRef) + { + // The argument is a pointer. We must dereference the pointer to get the value or object it points to. + t = t.GetElementType(); + if (t.IsValueType) + { + il.Emit(OpCodes.Ldobj, t); + } + else + { + il.Emit(OpCodes.Ldind_Ref); + } + anyByRef = true; + } + if (t.IsValueType) { il.Emit(OpCodes.Box, t); } - il.Emit(OpCodes.Callvirt, add); - il.Emit(OpCodes.Pop); + // args[c] = arg + il.Emit(OpCodes.Stelem_Ref); } il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Call, dispatch); + if (anyByRef) + { + // Dispatch() will have modified elements of the args list that correspond to out parameters. + for (var c = 0; c < signature.Length; c++) + { + Type t = signature[c]; + if (t.IsByRef) + { + t = t.GetElementType(); + // *arg = args[c] + il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); + il.Emit(OpCodes.Ldelem_Ref); + if (t.IsValueType) + { + il.Emit(OpCodes.Unbox_Any, t); + il.Emit(OpCodes.Stobj, t); + } + else + { + il.Emit(OpCodes.Stind_Ref); + } + } + } + } + if (method.ReturnType == voidtype) { il.Emit(OpCodes.Pop); @@ -218,7 +259,7 @@ public void Dispose() GC.SuppressFinalize(this); } - public object Dispatch(ArrayList args) + public object Dispatch(object[] args) { IntPtr gs = PythonEngine.AcquireLock(); object ob; @@ -235,7 +276,7 @@ public object Dispatch(ArrayList args) return ob; } - public object TrueDispatch(ArrayList args) + private object TrueDispatch(object[] args) { MethodInfo method = dtype.GetMethod("Invoke"); ParameterInfo[] pi = method.GetParameters(); @@ -259,20 +300,108 @@ public object TrueDispatch(ArrayList args) throw e; } - if (rtype == typeof(void)) + try { - return null; - } + int byRefCount = pi.Count(parameterInfo => parameterInfo.ParameterType.IsByRef); + if (byRefCount > 0) + { + // By symmetry with MethodBinder.Invoke, when there are out + // parameters we expect to receive a tuple containing + // the result, if any, followed by the out parameters. If there is only + // one out parameter and the return type of the method is void, + // we instead receive the out parameter as the result from Python. + + bool isVoid = rtype == typeof(void); + int tupleSize = byRefCount + (isVoid ? 0 : 1); + if (isVoid && byRefCount == 1) + { + // The return type is void and there is a single out parameter. + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (!Converter.ToManaged(op, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function did not return {t.GetElementType()} (the out parameter type)"); + throw new PythonException(); + } + args[i] = newArg; + break; + } + } + return null; + } + else if (Runtime.PyTuple_Check(op) && Runtime.PyTuple_Size(op) == tupleSize) + { + int index = isVoid ? 0 : 1; + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + IntPtr item = Runtime.PyTuple_GetItem(op, index++); + if (!Converter.ToManaged(item, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element {i} was not {t.GetElementType()} (the out parameter type)"); + throw new PythonException(); + } + args[i] = newArg; + } + } + if (isVoid) + { + return null; + } + IntPtr item0 = Runtime.PyTuple_GetItem(op, 0); + if (!Converter.ToManaged(item0, rtype, out object result0, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)"); + throw new PythonException(); + } + return result0; + } + else + { + string tpName = Runtime.PyObject_GetTypeName(op); + if (Runtime.PyTuple_Check(op)) + { + tpName += $" of size {Runtime.PyTuple_Size(op)}"; + } + StringBuilder sb = new StringBuilder(); + if (!isVoid) sb.Append(rtype.FullName); + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (sb.Length > 0) sb.Append(","); + sb.Append(t.GetElementType().FullName); + } + } + string returnValueString = isVoid ? "" : "the return value and "; + Exceptions.RaiseTypeError($"Expected a tuple ({sb}) of {returnValueString}the values for out and ref parameters, got {tpName}."); + throw new PythonException(); + } + } + + if (rtype == typeof(void)) + { + return null; + } - object result; - if (!Converter.ToManaged(op, rtype, out result, true)) + object result; + if (!Converter.ToManaged(op, rtype, out result, true)) + { + throw new PythonException(); + } + + return result; + } + finally { Runtime.XDecref(op); - throw new PythonException(); } - - Runtime.XDecref(op); - return result; } } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 3f879d3c4..034c1c3e8 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -960,34 +960,35 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i } // If there are out parameters, we return a tuple containing - // the result followed by the out parameters. If there is only + // the result, if any, followed by the out parameters. If there is only // one out parameter and the return type of the method is void, // we return the out parameter as the result to Python (for // code compatibility with ironpython). var mi = (MethodInfo)binding.info; - if (binding.outs == 1 && mi.ReturnType == typeof(void)) - { - } - if (binding.outs > 0) { ParameterInfo[] pi = mi.GetParameters(); int c = pi.Length; var n = 0; - IntPtr t = Runtime.PyTuple_New(binding.outs + 1); - IntPtr v = Converter.ToPython(result, mi.ReturnType); - Runtime.PyTuple_SetItem(t, n, v); - n++; + bool isVoid = mi.ReturnType == typeof(void); + int tupleSize = binding.outs + (isVoid ? 0 : 1); + IntPtr t = Runtime.PyTuple_New(tupleSize); + if (!isVoid) + { + IntPtr v = Converter.ToPython(result, mi.ReturnType); + Runtime.PyTuple_SetItem(t, n, v); + n++; + } for (var i = 0; i < c; i++) { Type pt = pi[i].ParameterType; - if (pi[i].IsOut || pt.IsByRef) + if (pt.IsByRef) { - v = Converter.ToPython(binding.args[i], pt.GetElementType()); + IntPtr v = Converter.ToPython(binding.args[i], pt.GetElementType()); Runtime.PyTuple_SetItem(t, n, v); n++; } @@ -995,7 +996,7 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding.outs == 1 && mi.ReturnType == typeof(void)) { - v = Runtime.PyTuple_GetItem(t, 1); + IntPtr v = Runtime.PyTuple_GetItem(t, 0); Runtime.XIncref(v); Runtime.XDecref(t); return v; diff --git a/src/testing/delegatetest.cs b/src/testing/delegatetest.cs index e2df9475f..ee66bdad7 100644 --- a/src/testing/delegatetest.cs +++ b/src/testing/delegatetest.cs @@ -13,6 +13,12 @@ namespace Python.Test public delegate bool BoolDelegate(); + public delegate void OutStringDelegate(out string value); + public delegate void RefStringDelegate(ref string value); + public delegate void OutIntDelegate(out int value); + public delegate void RefIntDelegate(ref int value); + public delegate void RefIntRefStringDelegate(ref int intValue, ref string stringValue); + public delegate int IntRefIntRefStringDelegate(ref int intValue, ref string stringValue); public class DelegateTest { @@ -27,6 +33,8 @@ public class DelegateTest public StringDelegate stringDelegate; public ObjectDelegate objectDelegate; public BoolDelegate boolDelegate; + public OutStringDelegate outStringDelegate; + public RefStringDelegate refStringDelegate; public DelegateTest() { @@ -42,6 +50,11 @@ public static string StaticSayHello() return "hello"; } + public void OutHello(out string value) + { + value = "hello"; + } + public string CallStringDelegate(StringDelegate d) { return d(); @@ -56,5 +69,35 @@ public bool CallBoolDelegate(BoolDelegate d) { return d(); } + + public void CallOutIntDelegate(OutIntDelegate d, out int value) + { + d(out value); + } + + public void CallRefIntDelegate(RefIntDelegate d, ref int value) + { + d(ref value); + } + + public void CallOutStringDelegate(OutStringDelegate d, out string value) + { + d(out value); + } + + public void CallRefStringDelegate(RefStringDelegate d, ref string value) + { + d(ref value); + } + + public void CallRefIntRefStringDelegate(RefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + d(ref intValue, ref stringValue); + } + + public int CallIntRefIntRefStringDelegate(IntRefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + return d(ref intValue, ref stringValue); + } } } diff --git a/src/testing/eventtest.cs b/src/testing/eventtest.cs index dfbd5c881..c9573f71a 100644 --- a/src/testing/eventtest.cs +++ b/src/testing/eventtest.cs @@ -7,7 +7,6 @@ namespace Python.Test /// public delegate void EventHandlerTest(object sender, EventArgsTest e); - #pragma warning disable 67 // Unused events, these are only accessed from Python public class EventTest { @@ -27,6 +26,10 @@ public class EventTest private event EventHandlerTest PrivateEvent; + public event OutStringDelegate OutStringEvent; + public event OutIntDelegate OutIntEvent; + public event RefStringDelegate RefStringEvent; + public event RefIntDelegate RefIntEvent; public static int s_value; public int value; @@ -77,6 +80,27 @@ protected static void OnProtectedStaticEvent(EventArgsTest e) } } + public void OnRefStringEvent(ref string data) + { + RefStringEvent?.Invoke(ref data); + } + + public void OnRefIntEvent(ref int data) + { + RefIntEvent?.Invoke(ref data); + } + + public void OnOutStringEvent(out string data) + { + data = default; + OutStringEvent?.Invoke(out data); + } + + public void OnOutIntEvent(out int data) + { + data = default; + OutIntEvent?.Invoke(out data); + } public void GenericHandler(object sender, EventArgsTest e) { @@ -88,6 +112,26 @@ public static void StaticHandler(object sender, EventArgsTest e) s_value = e.value; } + public void OutStringHandler(out string data) + { + data = value.ToString(); + } + + public void OutIntHandler(out int data) + { + data = value; + } + + public void RefStringHandler(ref string data) + { + data += "!"; + } + + public void RefIntHandler(ref int data) + { + data++; + } + public static void ShutUpCompiler() { // Quiet compiler warnings. diff --git a/src/tests/test_delegate.py b/src/tests/test_delegate.py index 909fd0f05..52ac8226d 100644 --- a/src/tests/test_delegate.py +++ b/src/tests/test_delegate.py @@ -276,6 +276,166 @@ def test_invalid_object_delegate(): with pytest.raises(TypeError): ob.CallObjectDelegate(d) +def test_out_int_delegate(): + """Test delegate with an out int parameter.""" + from Python.Test import OutIntDelegate + value = 7 + + def out_hello_func(ignored): + return 5 + + d = OutIntDelegate(out_hello_func) + result = d(value) + assert result == 5 + + ob = DelegateTest() + result = ob.CallOutIntDelegate(d, value) + assert result == 5 + + def invalid_handler(ignored): + return '5' + + d = OutIntDelegate(invalid_handler) + with pytest.raises(TypeError): + result = d(value) + +def test_out_string_delegate(): + """Test delegate with an out string parameter.""" + from Python.Test import OutStringDelegate + value = 'goodbye' + + def out_hello_func(ignored): + return 'hello' + + d = OutStringDelegate(out_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallOutStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefIntDelegate + value = 7 + + def ref_hello_func(data): + assert data == value + return data + 1 + + d = RefIntDelegate(ref_hello_func) + result = d(value) + assert result == value + 1 + + ob = DelegateTest() + result = ob.CallRefIntDelegate(d, value) + assert result == value + 1 + +def test_ref_string_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefStringDelegate + value = 'goodbye' + + def ref_hello_func(data): + assert data == value + return 'hello' + + d = RefStringDelegate(ref_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallRefStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import RefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + 1, stringValue + '!') + + d = RefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = RefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = RefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def long_tuple(intValue, stringValue): + return (5, 'a', 'b') + + d = RefIntRefStringDelegate(long_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_tuple_item(intValue, stringValue): + return ('a', 'b') + + d = RefIntRefStringDelegate(wrong_tuple_item) + with pytest.raises(TypeError): + result = d(intData, stringData) + +def test_int_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import IntRefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + len(stringValue), intValue + 1, stringValue + '!') + + d = IntRefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallIntRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = IntRefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = IntRefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_return_type(intValue, stringValue): + return ('a', 7, 'b') + + d = IntRefIntRefStringDelegate(wrong_return_type) + with pytest.raises(TypeError): + result = d(intData, stringData) + # test async delegates # test multicast delegates diff --git a/src/tests/test_event.py b/src/tests/test_event.py index e9c0ffd8a..885589032 100644 --- a/src/tests/test_event.py +++ b/src/tests/test_event.py @@ -295,6 +295,41 @@ def handler(sender, args, dict_=dict_): ob.OnPublicEvent(EventArgsTest(20)) assert dict_['value'] == 10 +def test_out_function_handler(): + """Test function handlers with Out arguments.""" + ob = EventTest() + + value = 10 + def handler(ignored): + return value + + ob.OutIntEvent += handler + result = ob.OnOutIntEvent(55) + assert result == value + + ob.OutStringEvent += handler + value = 'This is the event data' + result = ob.OnOutStringEvent('Hello') + assert result == value + +def test_ref_function_handler(): + """Test function handlers with Ref arguments.""" + ob = EventTest() + + value = 10 + def handler(data): + return value + data + + ob.RefIntEvent += ob.RefIntHandler + ob.RefIntEvent += handler + result = ob.OnRefIntEvent(5) + assert result == value + 5 + 1 + + ob.RefStringEvent += ob.RefStringHandler + ob.RefStringEvent += handler + value = 'This is the event data' + result = ob.OnRefStringEvent('!') + assert result == value + '!!' def test_add_non_callable_handler(): """Test handling of attempts to add non-callable handlers.""" From 909ed1f7a53e151913d993e53bf6a454030d2f12 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 30 Nov 2020 14:05:05 -0800 Subject: [PATCH 020/151] dropped net40 target from modern projects --- CHANGELOG.md | 2 ++ src/console/pythonconsole.cs | 2 +- src/tests/tests.pyproj | 10 ++-------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d516488..110d7c1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,8 @@ when .NET expects an integer [#1342][i1342] ### Removed - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) +- support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 +(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) ## [2.5.0][] - 2020-06-14 diff --git a/src/console/pythonconsole.cs b/src/console/pythonconsole.cs index 912e9bb0d..bf17848f7 100644 --- a/src/console/pythonconsole.cs +++ b/src/console/pythonconsole.cs @@ -26,7 +26,7 @@ private PythonConsole() [STAThread] public static int Main(string[] args) { - // Only net40 is capable to safely inject python.runtime.dll into resources. + // Only .NET Framework is capable to safely inject python.runtime.dll into resources. #if NET40 // reference the static assemblyLoader to stop it being optimized away AssemblyLoader a = assemblyLoader; diff --git a/src/tests/tests.pyproj b/src/tests/tests.pyproj index 4bdbc6b14..439bc856a 100644 --- a/src/tests/tests.pyproj +++ b/src/tests/tests.pyproj @@ -14,14 +14,8 @@ - - - - - - - - + + 10.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets From 47e926ed39fec384a4b61d298285f86e40a0bc34 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 1 Dec 2020 18:22:36 -0800 Subject: [PATCH 021/151] use .NET Standard 2.0 platform detection features --- src/runtime/platform/NativeCodePage.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs index b6fe89425..bf65240a5 100644 --- a/src/runtime/platform/NativeCodePage.cs +++ b/src/runtime/platform/NativeCodePage.cs @@ -8,17 +8,20 @@ class NativeCodePageHelper { /// /// Initialized by InitializeNativeCodePage. - /// + /// /// This points to a page of memory allocated using mmap or VirtualAlloc /// (depending on the system), and marked read and execute (not write). /// Very much on purpose, the page is *not* released on a shutdown and /// is instead leaked. See the TestDomainReload test case. - /// + /// + /// /// The contents of the page are two native functions: one that returns 0, /// one that returns 1. - /// + /// + /// /// If python didn't keep its gc list through a Py_Finalize we could remove /// this entire section. + /// /// internal static IntPtr NativeCodePage = IntPtr.Zero; @@ -95,10 +98,11 @@ public static NativeCode Active }; /// - /// Code for X86. - /// + /// Code for X86. + /// /// It's bitwise identical to X86_64, so we just point to it. /// + /// /// public static readonly NativeCode I386 = X86_64; } @@ -145,7 +149,7 @@ public void SetReadExec(IntPtr mappedMemory, int numBytes) } } - class UnixMemoryMapper : IMemoryMapper + class PosixMemoryMapper : IMemoryMapper { const int PROT_READ = 0x1; const int PROT_WRITE = 0x2; @@ -196,7 +200,7 @@ internal static IMemoryMapper CreateMemoryMapper() else { // Linux, OSX, FreeBSD - return new UnixMemoryMapper(); + return new PosixMemoryMapper(); } } From 21683b3fa34f18828c402e51aacc05930a46d373 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 1 Dec 2020 18:26:02 -0800 Subject: [PATCH 022/151] drop NativeCodePage alltogether --- src/runtime/platform/NativeCodePage.cs | 230 ------------------------- src/runtime/platform/Types.cs | 192 --------------------- 2 files changed, 422 deletions(-) delete mode 100644 src/runtime/platform/NativeCodePage.cs delete mode 100644 src/runtime/platform/Types.cs diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs deleted file mode 100644 index bf65240a5..000000000 --- a/src/runtime/platform/NativeCodePage.cs +++ /dev/null @@ -1,230 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace Python.Runtime.Platform -{ - class NativeCodePageHelper - { - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X86: - return I386; - case Architecture.X64: - return X86_64; - default: - return null; - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class PosixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - return 0x20; - } - else - { - // OSX, FreeBSD - return 0x1000; - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return new WindowsMemoryMapper(); - } - else - { - // Linux, OSX, FreeBSD - return new PosixMemoryMapper(); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } - } -} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs deleted file mode 100644 index 15235da5a..000000000 --- a/src/runtime/platform/Types.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace Python.Runtime.Platform -{ - public enum MachineType - { - i386, - x86_64, - armv7l, - armv8, - aarch64, - Other - }; - - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - - static class SystemInfo - { - public static MachineType GetMachineType() - { - return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); - } - - public static string GetArchitecture() - { - return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); - } - - public static OperatingSystemType GetSystemType() - { - if (Runtime.IsWindows) - { - return OperatingSystemType.Windows; - } - switch (PythonEngine.Platform) - { - case "linux": - return OperatingSystemType.Linux; - - case "darwin": - return OperatingSystemType.Darwin; - - default: - return OperatingSystemType.Other; - } - } - - #region WINDOWS - - static string GetArchName_Windows() - { - // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details - return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); - } - - static MachineType GetMachineType_Windows() - { - if (Runtime.Is32Bit) - { - return MachineType.i386; - } - switch (GetArchName_Windows()) - { - case "AMD64": - return MachineType.x86_64; - case "ARM64": - return MachineType.aarch64; - default: - return MachineType.Other; - } - } - - #endregion - - #region UNIX - - - [StructLayout(LayoutKind.Sequential)] - unsafe struct utsname_linux - { - const int NameLength = 65; - - /* Name of the implementation of the operating system. */ - public fixed byte sysname[NameLength]; - - /* Name of this node on the network. */ - public fixed byte nodename[NameLength]; - - /* Current release level of this implementation. */ - public fixed byte release[NameLength]; - - /* Current version level of this release. */ - public fixed byte version[NameLength]; - - /* Name of the hardware type the system is running on. */ - public fixed byte machine[NameLength]; - - // GNU extension - fixed byte domainname[NameLength]; /* NIS or YP domain name */ - } - - [StructLayout(LayoutKind.Sequential)] - unsafe struct utsname_darwin - { - const int NameLength = 256; - - /* Name of the implementation of the operating system. */ - public fixed byte sysname[NameLength]; - - /* Name of this node on the network. */ - public fixed byte nodename[NameLength]; - - /* Current release level of this implementation. */ - public fixed byte release[NameLength]; - - /* Current version level of this release. */ - public fixed byte version[NameLength]; - - /* Name of the hardware type the system is running on. */ - public fixed byte machine[NameLength]; - } - - [DllImport("libc")] - static extern int uname(IntPtr buf); - - - static unsafe string GetArchName_Unix() - { - switch (GetSystemType()) - { - case OperatingSystemType.Linux: - { - var buf = stackalloc utsname_linux[1]; - if (uname((IntPtr)buf) != 0) - { - return null; - } - return Marshal.PtrToStringAnsi((IntPtr)buf->machine); - } - - case OperatingSystemType.Darwin: - { - var buf = stackalloc utsname_darwin[1]; - if (uname((IntPtr)buf) != 0) - { - return null; - } - return Marshal.PtrToStringAnsi((IntPtr)buf->machine); - } - - default: - return null; - } - } - - static unsafe MachineType GetMachineType_Unix() - { - switch (GetArchName_Unix()) - { - case "x86_64": - case "em64t": - return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; - case "i386": - case "i686": - return MachineType.i386; - - case "armv7l": - return MachineType.armv7l; - case "armv8": - return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; - case "aarch64": - return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; - - default: - return MachineType.Other; - } - } - - #endregion - } -} From 972c41d018c78f08d2531c8e3ae640f3038e79d0 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 4 Dec 2020 15:25:43 -0800 Subject: [PATCH 023/151] WIP: use C# 9 function pointers for PInvoke --- src/runtime/BorrowedReference.cs | 3 +- src/runtime/NewReference.cs | 10 + src/runtime/Python.Runtime.csproj | 1 + src/runtime/classmanager.cs | 26 +- src/runtime/classobject.cs | 4 +- src/runtime/importhook.cs | 131 ++- src/runtime/interop.cs | 3 +- src/runtime/managedtype.cs | 4 + src/runtime/moduleobject.cs | 26 +- src/runtime/pydict.cs | 11 +- src/runtime/pyscope.cs | 32 +- src/runtime/pythonengine.cs | 68 +- src/runtime/runtime.cs | 1688 +++++++++++++++++++---------- src/runtime/runtime_data.cs | 4 +- src/runtime/runtime_state.cs | 78 +- src/runtime/typemanager.cs | 56 +- 16 files changed, 1345 insertions(+), 800 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index a9ea327e9..2f5c347c7 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,10 +10,11 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; - /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; + /// Gets a raw pointer to the Python object + public IntPtr DangerousGetAddressOrNull() => this.pointer; /// /// Creates new instance of from raw pointer. Unsafe. diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index a4ed75918..b9d2602f7 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -28,6 +28,16 @@ public PyObject MoveToPyObject() return result; } + /// Moves ownership of this instance to unmanged pointer + public IntPtr DangerousMoveToPointer() + { + if (this.IsNull()) throw new NullReferenceException(); + + var result = this.pointer; + this.pointer = IntPtr.Zero; + return result; + } + /// Moves ownership of this instance to unmanged pointer public IntPtr DangerousMoveToPointerOrNull() { diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 4939fae34..a09b0ff78 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -2,6 +2,7 @@ netstandard2.0 AnyCPU + 9.0 Python.Runtime Python.Runtime 9.0 diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0cbff371f..1ee06e682 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -117,7 +117,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // Python object's dictionary tool; thus raising an AttributeError // instead of a TypeError. // Classes are re-initialized on in RestoreRuntimeData. - IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + var dict = new BorrowedReference(Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict)); foreach (var member in cls.Value.dotNetMembers) { // No need to decref the member, the ClassBase instance does @@ -269,7 +269,7 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr tp = TypeManager.GetTypeHandle(impl, type); // Finally, initialize the class __dict__ and return the object. - IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + var dict = new BorrowedReference(Marshal.ReadIntPtr(tp, TypeOffset.tp_dict)); if (impl.dotNetMembers == null) @@ -282,7 +282,7 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; impl.dotNetMembers.Add(name); - Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + Runtime.PyDict_SetItemString(dict, name, item.ObjectReference); // Decref the item now that it's been used. item.DecrRefCount(); if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp)) { @@ -291,20 +291,15 @@ private static void InitClassBase(Type type, ClassBase impl) } // If class has constructors, generate an __doc__ attribute. - IntPtr doc = IntPtr.Zero; + NewReference doc = default; Type marker = typeof(DocStringAttribute); var attrs = (Attribute[])type.GetCustomAttributes(marker, false); - if (attrs.Length == 0) - { - doc = IntPtr.Zero; - } - else + if (attrs.Length != 0) { var attr = (DocStringAttribute)attrs[0]; string docStr = attr.DocString; - doc = Runtime.PyString_FromString(docStr); + doc = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docStr)); Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); - Runtime.XDecref(doc); } var co = impl as ClassObject; @@ -320,20 +315,21 @@ private static void InitClassBase(Type type, ClassBase impl) var ctors = new ConstructorBinding(type, tp, co.binder); // ExtensionType types are untracked, so don't Incref() them. // TODO: deprecate __overloads__ soon... - Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.pyHandle); - Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.pyHandle); + Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.ObjectReference); + Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.ObjectReference); ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. - if (!CLRModule._SuppressDocs && doc == IntPtr.Zero) + if (!CLRModule._SuppressDocs && doc.IsNull()) { doc = co.GetDocString(); Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); - Runtime.XDecref(doc); } } } + doc.Dispose(); + // The type has been modified after PyType_Ready has been called // Refresh the type Runtime.PyType_Modified(tp); diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 826ae5c54..4aa97f648 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -31,7 +31,7 @@ internal ClassObject(Type tp) : base(tp) /// /// Helper to get docstring from reflected constructor info. /// - internal IntPtr GetDocString() + internal NewReference GetDocString() { MethodBase[] methods = binder.GetMethods(); var str = ""; @@ -43,7 +43,7 @@ internal IntPtr GetDocString() } str += t.ToString(); } - return Runtime.PyString_FromString(str); + return NewReference.DangerousFromPointer(Runtime.PyString_FromString(str)); } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index af6174188..8d1a60342 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -13,6 +13,7 @@ internal static class ImportHook private static CLRModule root; private static MethodWrapper hook; private static IntPtr py_clr_module; + static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); private static IntPtr module_def = IntPtr.Zero; @@ -74,7 +75,7 @@ static void RestoreImport() /// /// Initialization performed on startup of the Python runtime. /// - internal static void Initialize() + internal static unsafe void Initialize() { InitImport(); @@ -86,14 +87,13 @@ internal static void Initialize() py_clr_module = Runtime.PyModule_Create2(module_def, 3); // both dicts are borrowed references - IntPtr mod_dict = Runtime.PyModule_GetDict(py_clr_module); - IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** - clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); Runtime.PyDict_Update(mod_dict, clr_dict); - IntPtr dict = Runtime.PyImport_GetModuleDict(); - Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); - Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); } @@ -143,67 +143,62 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) /// /// Return the clr python module (new reference) /// - public static IntPtr GetCLRModule(IntPtr? fromList = null) + public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) { root.InitializePreload(); // update the module dictionary with the contents of the root dictionary root.LoadNames(); - IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module); - IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** - clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); Runtime.PyDict_Update(py_mod_dict, clr_dict); // find any items from the from list and get them from the root if they're not // already in the module dictionary - if (fromList != null && fromList != IntPtr.Zero) + if (fromList != null && fromList != default) { - if (Runtime.PyTuple_Check(fromList.GetValueOrDefault())) + if (Runtime.PyTuple_Check(fromList)) { - Runtime.XIncref(py_mod_dict); - using (var mod_dict = new PyDict(py_mod_dict)) + using var mod_dict = new PyDict(py_mod_dict); + using var from = new PyTuple(fromList); + foreach (PyObject item in from) { - Runtime.XIncref(fromList.GetValueOrDefault()); - using (var from = new PyTuple(fromList.GetValueOrDefault())) + if (mod_dict.HasKey(item)) { - foreach (PyObject item in from) - { - if (mod_dict.HasKey(item)) - { - continue; - } - - var s = item.AsManagedObject(typeof(string)) as string; - if (s == null) - { - continue; - } - - ManagedType attr = root.GetAttribute(s, true); - if (attr == null) - { - continue; - } - - Runtime.XIncref(attr.pyHandle); - using (var obj = new PyObject(attr.pyHandle)) - { - mod_dict.SetItem(s, obj); - } - } + continue; + } + + var s = item.AsManagedObject(typeof(string)) as string; + if (s == null) + { + continue; + } + + ManagedType attr = root.GetAttribute(s, true); + if (attr == null) + { + continue; + } + + Runtime.XIncref(attr.pyHandle); + using (var obj = new PyObject(attr.pyHandle)) + { + mod_dict.SetItem(s, obj); } } } } Runtime.XIncref(py_clr_module); - return py_clr_module; + return NewReference.DangerousFromPointer(py_clr_module); } /// /// The actual import hook that ties Python to the managed world. /// - public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) + public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) { + var args = new BorrowedReference(argsRaw); + // Replacement for the builtin __import__. The original import // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. @@ -214,9 +209,8 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); } - // borrowed reference - IntPtr py_mod_name = Runtime.PyTuple_GetItem(args, 0); - if (py_mod_name == IntPtr.Zero || + BorrowedReference py_mod_name = Runtime.PyTuple_GetItem(args, 0); + if (py_mod_name.IsNull || !Runtime.IsStringType(py_mod_name)) { return Exceptions.RaiseTypeError("string expected"); @@ -225,12 +219,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // Check whether the import is of the form 'from x import y'. // This determines whether we return the head or tail module. - IntPtr fromList = IntPtr.Zero; + BorrowedReference fromList = default; var fromlist = false; if (num_args >= 4) { fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != IntPtr.Zero && + if (fromList != default && Runtime.PyObject_IsTrue(fromList) == 1) { fromlist = true; @@ -243,16 +237,16 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // the module. if (mod_name == "clr") { - IntPtr clr_module = GetCLRModule(fromList); - if (clr_module != IntPtr.Zero) + NewReference clr_module = GetCLRModule(fromList); + if (!clr_module.IsNull()) { - IntPtr sys_modules = Runtime.PyImport_GetModuleDict(); - if (sys_modules != IntPtr.Zero) + BorrowedReference sys_modules = Runtime.PyImport_GetModuleDict(); + if (!sys_modules.IsNull) { Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); } } - return clr_module; + return clr_module.DangerousMoveToPointerOrNull(); } string realname = mod_name; @@ -265,7 +259,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // Turns out that the AssemblyManager.ResolveHandler() checks to see if any // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args, kw); + IntPtr res = Runtime.PyObject_Call(py_import, args.DangerousGetAddress(), kw); if (res != IntPtr.Zero) { // There was no error. @@ -300,10 +294,10 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // See if sys.modules for this interpreter already has the // requested module. If so, just return the existing module. - IntPtr modules = Runtime.PyImport_GetModuleDict(); - IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name); + BorrowedReference modules = Runtime.PyImport_GetModuleDict(); + BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name); - if (module != IntPtr.Zero) + if (module != default) { if (fromlist) { @@ -312,16 +306,15 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) var mod = ManagedType.GetManagedObject(module) as ModuleObject; mod?.LoadNames(); } - Runtime.XIncref(module); - return module; + return Runtime.NewRef(module).DangerousMoveToPointer(); } if (clr_prefix != null) { - return GetCLRModule(fromList); + return GetCLRModule(fromList).DangerousMoveToPointerOrNull(); } module = Runtime.PyDict_GetItemString(modules, names[0]); - Runtime.XIncref(module); - return module; + return Runtime.NewRefOrNull(module) + .DangerousMoveToPointer(); } Exceptions.Clear(); @@ -358,12 +351,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } // Add the module to sys.modules - Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.pyHandle); + Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); // If imported from CLR add clr. to sys.modules as well if (clr_prefix != null) { - Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.pyHandle); + Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.ObjectReference); } } @@ -380,7 +373,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } } - private static bool IsLoadAll(IntPtr fromList) + private static bool IsLoadAll(BorrowedReference fromList) { if (CLRModule.preload) { @@ -390,10 +383,8 @@ private static bool IsLoadAll(IntPtr fromList) { return false; } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + using var fp = Runtime.PySequence_GetItem(fromList, 0); + return Runtime.GetManagedString(fp) == "*"; } } } diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index ff9a98622..0f56f77d9 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -206,8 +206,9 @@ public static int Size(IntPtr pyType) public static int ob_type; private static readonly int size; - private static bool IsException(IntPtr pyObject) + private static bool IsException(IntPtr pyObjectPtr) { + var pyObject = new BorrowedReference(pyObjectPtr); var type = Runtime.PyObject_TYPE(pyObject); return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index b4baef835..09e8a3be2 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -27,6 +27,8 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); + private static readonly Dictionary _managedObjs = new Dictionary(); internal void IncrRefCount() @@ -138,6 +140,8 @@ internal static ManagedType GetManagedObjectErr(IntPtr ob) } + internal static bool IsManagedType(BorrowedReference ob) + => IsManagedType(ob.DangerousGetAddressOrNull()); internal static bool IsManagedType(IntPtr ob) { if (ob != IntPtr.Zero) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 07dd20e55..3fdd99b9a 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -18,6 +18,7 @@ internal class ModuleObject : ExtensionType internal string moduleName; internal IntPtr dict; + internal BorrowedReference DictRef => new BorrowedReference(dict); protected string _namespace; public ModuleObject(string name) @@ -44,17 +45,14 @@ public ModuleObject(string name) } dict = Runtime.PyDict_New(); - IntPtr pyname = Runtime.PyString_FromString(moduleName); - IntPtr pyfilename = Runtime.PyString_FromString(filename); - IntPtr pydocstring = Runtime.PyString_FromString(docstring); - IntPtr pycls = TypeManager.GetTypeHandle(GetType()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname); - Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename); - Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring); - Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls); - Runtime.XDecref(pyname); - Runtime.XDecref(pyfilename); - Runtime.XDecref(pydocstring); + using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); + using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); + using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); + BorrowedReference pycls = TypeManager.GetTypeReference(GetType()); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__name__, pyname); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__file__, pyfilename); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__doc__, pydocstring); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__class__, pycls); Runtime.XIncref(dict); SetObjectDict(pyHandle, dict); @@ -177,9 +175,9 @@ public void LoadNames() { continue; } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); + BorrowedReference attr = Runtime.PyDict_GetItemString(DictRef, name); // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) + if (!attr.IsNull) { continue; } @@ -344,7 +342,7 @@ protected override void OnSave(InterDomainContext context) // destroy the cache(s) foreach (var pair in cache) { - if ((Runtime.PyDict_DelItemString(dict, pair.Key) == -1) && + if ((Runtime.PyDict_DelItemString(DictRef, pair.Key) == -1) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { // Trying to remove a key that's not in the dictionary diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index ade873f7a..4b850a9f9 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -22,6 +22,7 @@ public PyDict(IntPtr ptr) : base(ptr) { } + internal PyDict(BorrowedReference reference) : base(reference) { } /// /// PyDict Constructor @@ -103,12 +104,12 @@ public bool HasKey(string key) /// public PyObject Keys() { - IntPtr items = Runtime.PyDict_Keys(obj); - if (items == IntPtr.Zero) + using var items = Runtime.PyDict_Keys(Reference); + if (items.IsNull()) { throw new PythonException(); } - return new PyObject(items); + return items.MoveToPyObject(); } @@ -137,7 +138,7 @@ public PyObject Values() /// public PyObject Items() { - var items = Runtime.PyDict_Items(this.obj); + var items = Runtime.PyDict_Items(this.Reference); try { if (items.IsNull()) @@ -179,7 +180,7 @@ public PyDict Copy() /// public void Update(PyObject other) { - int result = Runtime.PyDict_Update(obj, other.obj); + int result = Runtime.PyDict_Update(Reference, other.Reference); if (result < 0) { throw new PythonException(); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index d61573733..72cb9f247 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -30,11 +30,13 @@ public class PyScope : DynamicObject, IDisposable /// the python Module object the scope associated with. /// internal IntPtr obj; + internal BorrowedReference Reference => new BorrowedReference(obj); /// /// the variable dict of the scope. /// internal readonly IntPtr variables; + internal BorrowedReference VarsRef => new BorrowedReference(variables); private bool _isDisposed; private bool _finalized = false; @@ -56,20 +58,20 @@ public class PyScope : DynamicObject, IDisposable /// /// Create a scope based on a Python Module. /// - internal PyScope(IntPtr ptr, PyScopeManager manager) + internal PyScope(ref NewReference ptr, PyScopeManager manager) { if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(ptr), Runtime.PyModuleType)) { throw new PyScopeException("object is not a module"); } Manager = manager ?? PyScopeManager.Global; - obj = ptr; + obj = ptr.DangerousMoveToPointer(); //Refcount of the variables not increase - variables = Runtime.PyModule_GetDict(obj); + variables = Runtime.PyModule_GetDict(Reference).DangerousGetAddress(); PythonException.ThrowIfIsNull(variables); int res = Runtime.PyDict_SetItem( - variables, PyIdentifier.__builtins__, + VarsRef, PyIdentifier.__builtins__, Runtime.PyEval_GetBuiltins() ); PythonException.ThrowIfIsNotZero(res); @@ -184,7 +186,7 @@ public void ImportAll(string name) /// public void ImportAll(PyScope scope) { - int result = Runtime.PyDict_Update(variables, scope.variables); + int result = Runtime.PyDict_Update(VarsRef, scope.VarsRef); if (result < 0) { throw new PythonException(); @@ -203,8 +205,8 @@ public void ImportAll(PyObject module) { throw new PyScopeException("object is not a module"); } - var module_dict = Runtime.PyModule_GetDict(module.obj); - int result = Runtime.PyDict_Update(variables, module_dict); + var module_dict = Runtime.PyModule_GetDict(module.Reference); + int result = Runtime.PyDict_Update(VarsRef, module_dict); if (result < 0) { throw new PythonException(); @@ -219,7 +221,7 @@ public void ImportAll(PyObject module) /// public void ImportAll(PyDict dict) { - int result = Runtime.PyDict_Update(variables, dict.obj); + int result = Runtime.PyDict_Update(VarsRef, dict.Reference); if (result < 0) { throw new PythonException(); @@ -277,10 +279,10 @@ public T Execute(PyObject script, PyDict locals = null) public PyObject Eval(string code, PyDict locals = null) { Check(); - IntPtr _locals = locals == null ? variables : locals.obj; + BorrowedReference _locals = locals == null ? VarsRef : locals.Reference; NewReference reference = Runtime.PyRun_String( - code, RunFlagType.Eval, variables, _locals + code, RunFlagType.Eval, VarsRef, _locals ); PythonException.ThrowIfIsNull(reference); return reference.MoveToPyObject(); @@ -310,11 +312,11 @@ public T Eval(string code, PyDict locals = null) public void Exec(string code, PyDict locals = null) { Check(); - IntPtr _locals = locals == null ? variables : locals.obj; - Exec(code, variables, _locals); + BorrowedReference _locals = locals == null ? VarsRef : locals.Reference; + Exec(code, VarsRef, _locals); } - private void Exec(string code, IntPtr _globals, IntPtr _locals) + private void Exec(string code, BorrowedReference _globals, BorrowedReference _locals) { NewReference reference = Runtime.PyRun_String( code, RunFlagType.File, _globals, _locals @@ -551,11 +553,11 @@ internal PyScope NewScope(string name) name = ""; } var module = Runtime.PyModule_New(name); - if (module == IntPtr.Zero) + if (module.IsNull()) { throw new PythonException(); } - return new PyScope(module, this); + return new PyScope(ref module, this); } /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 781d345e7..b5334fabc 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -211,15 +211,15 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + NewReference clr = Python.Runtime.ImportHook.GetCLRModule(); + BorrowedReference clr_dict = Runtime.PyModule_GetDict(clr); var locals = new PyDict(); try { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); + BorrowedReference module = Runtime.PyImport_AddModule("clr._extras"); + BorrowedReference module_globals = Runtime.PyModule_GetDict(module); + BorrowedReference builtins = Runtime.PyEval_GetBuiltins(); Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); @@ -228,7 +228,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, { // add the contents of clr.py to the module string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); + Exec(clr_py, module_globals, locals.Reference); } // add the imported module to the clr module, and copy the API functions @@ -240,7 +240,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + Runtime.PyDict_SetItem(clr_dict, key.Reference, value.Reference); value.Dispose(); } key.Dispose(); @@ -305,7 +305,8 @@ public static IntPtr InitExt() return IntPtr.Zero; } - return Python.Runtime.ImportHook.GetCLRModule(); + return Python.Runtime.ImportHook.GetCLRModule() + .DangerousMoveToPointerOrNull(); } /// @@ -544,7 +545,9 @@ public static PyObject Compile(string code, string filename = "", RunFlagType mo /// public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals = null) { - PyObject result = RunString(code, globals, locals, RunFlagType.Eval); + var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); + var localsRef = new BorrowedReference(locals.GetValueOrDefault()); + PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.Eval); return result; } @@ -558,12 +561,27 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) { - using (PyObject result = RunString(code, globals, locals, RunFlagType.File)) + var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); + var localsRef = new BorrowedReference(locals.GetValueOrDefault()); + using PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.File); + if (result.obj != Runtime.PyNone) { - if (result.obj != Runtime.PyNone) - { - throw new PythonException(); - } + throw new PythonException(); + } + } + /// + /// Exec Method + /// + /// + /// Run a string containing Python code. + /// It's a subset of Python exec function. + /// + internal static void Exec(string code, BorrowedReference globals, BorrowedReference locals = default) + { + using PyObject result = RunString(code, globals: globals, locals: locals, RunFlagType.File); + if (result.obj != Runtime.PyNone) + { + throw new PythonException(); } } @@ -599,7 +617,7 @@ public static int Interrupt(ulong pythonThreadID) [Obsolete("RunString is deprecated and will be removed. Use Exec/Eval/RunSimpleString instead.")] public static PyObject RunString(string code, IntPtr? globals = null, IntPtr? locals = null) { - return RunString(code, globals, locals, RunFlagType.File); + return RunString(code, new BorrowedReference(globals.GetValueOrDefault()), new BorrowedReference(locals.GetValueOrDefault()), RunFlagType.File); } /// @@ -610,20 +628,19 @@ public static PyObject RunString(string code, IntPtr? globals = null, IntPtr? lo /// executing the code string as a PyObject instance, or null if /// an exception was raised. /// - internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, RunFlagType flag) + internal static PyObject RunString(string code, BorrowedReference globals, BorrowedReference locals, RunFlagType flag) { - var borrowedGlobals = true; - if (globals == null) + NewReference tempGlobals = default; + if (globals.IsNull) { globals = Runtime.PyEval_GetGlobals(); - if (globals == IntPtr.Zero) + if (globals.IsNull) { - globals = Runtime.PyDict_New(); + globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New()); Runtime.PyDict_SetItem( - globals.Value, PyIdentifier.__builtins__, + globals, PyIdentifier.__builtins__, Runtime.PyEval_GetBuiltins() ); - borrowedGlobals = false; } } @@ -635,17 +652,14 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { NewReference result = Runtime.PyRun_String( - code, flag, globals.Value, locals.Value + code, flag, globals, locals ); PythonException.ThrowIfIsNull(result); return result.MoveToPyObject(); } finally { - if (!borrowedGlobals) - { - Runtime.XDecref(globals.Value); - } + tempGlobals.Dispose(); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index c754b80b5..889af1565 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -17,7 +17,7 @@ namespace Python.Runtime /// the responsibility of the caller to have acquired the GIL /// before calling any of these methods. /// - public class Runtime + public unsafe class Runtime { public static int UCS => _UCS; internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; @@ -481,8 +481,8 @@ private static void ClearClrModules() for (long i = 0; i < length; i++) { var item = PyList_GetItem(items, i); - var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); - var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); if (ManagedType.IsManagedType(module)) { PyDict_DelItem(modules, name); @@ -498,7 +498,7 @@ private static void RemoveClrRootModule() PyDictTryDelItem(modules, "clr._extra"); } - private static void PyDictTryDelItem(IntPtr dict, string key) + private static void PyDictTryDelItem(BorrowedReference dict, string key) { if (PyDict_DelItemString(dict, key) == 0) { @@ -528,7 +528,7 @@ private static void MoveClrInstancesOnwershipToPython() obj.CallTypeClear(); // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), // thus just be safe to give it back to GC chain. - if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + if (!_PyObject_GC_IS_TRACKED(obj.ObjectReference)) { PyObject_GC_Track(obj.pyHandle); } @@ -722,6 +722,15 @@ internal static unsafe void XIncref(IntPtr op) #endif } + internal static NewReference NewRef(BorrowedReference reference) + { + var address = reference.DangerousGetAddress(); + XIncref(address); + return NewReference.DangerousFromPointer(address); + } + internal static NewReference NewRefOrNull(BorrowedReference reference) + => reference.IsNull ? default : NewRef(reference); + /// /// Increase Python's ref counter for the given object, and get the object back. /// @@ -788,146 +797,147 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_IncRef(IntPtr ob); + + internal static void Py_IncRef(IntPtr ob) => Delegates.Py_IncRef(ob); /// /// Export of Macro Py_XDecRef. Use XDecref instead. /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_DecRef(IntPtr ob); + + internal static void Py_DecRef(IntPtr ob) => Delegates.Py_DecRef(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_Initialize(); + + internal static void Py_Initialize() => Delegates.Py_Initialize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); + + internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_IsInitialized(); + + internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_Finalize(); + + internal static void Py_Finalize() => Delegates.Py_Finalize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_NewInterpreter(); + + internal static IntPtr Py_NewInterpreter() => Delegates.Py_NewInterpreter(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_EndInterpreter(IntPtr threadState); + + internal static void Py_EndInterpreter(IntPtr threadState) => Delegates.Py_EndInterpreter(threadState); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_New(IntPtr istate); + + internal static IntPtr PyThreadState_New(IntPtr istate) => Delegates.PyThreadState_New(istate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_Get(); + + internal static IntPtr PyThreadState_Get() => Delegates.PyThreadState_Get(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyThreadState_UncheckedGet(); + + internal static IntPtr _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThread_get_key_value(IntPtr key); + + internal static IntPtr PyThread_get_key_value(IntPtr key) => Delegates.PyThread_get_key_value(key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThread_get_thread_ident(); + + internal static int PyThread_get_thread_ident() => Delegates.PyThread_get_thread_ident(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThread_set_key_value(IntPtr key, IntPtr value); + + internal static int PyThread_set_key_value(IntPtr key, IntPtr value) => Delegates.PyThread_set_key_value(key, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_Swap(IntPtr key); + + internal static IntPtr PyThreadState_Swap(IntPtr key) => Delegates.PyThreadState_Swap(key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGILState_Ensure(); + + internal static IntPtr PyGILState_Ensure() => Delegates.PyGILState_Ensure(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyGILState_Release(IntPtr gs); + + internal static void PyGILState_Release(IntPtr gs) => Delegates.PyGILState_Release(gs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGILState_GetThisThreadState(); + + internal static IntPtr PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - public static extern int Py_Main( + + public static int Py_Main( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv - ); + ) => Delegates.Py_Main(argc, argv +); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_InitThreads(); + + internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyEval_ThreadsInitialized(); + + internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_AcquireLock(); + + internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_ReleaseLock(); + + internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_AcquireThread(IntPtr tstate); + + internal static void PyEval_AcquireThread(IntPtr tstate) => Delegates.PyEval_AcquireThread(tstate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_ReleaseThread(IntPtr tstate); + + internal static void PyEval_ReleaseThread(IntPtr tstate) => Delegates.PyEval_ReleaseThread(tstate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_SaveThread(); + + internal static IntPtr PyEval_SaveThread() => Delegates.PyEval_SaveThread(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_RestoreThread(IntPtr tstate); + + internal static void PyEval_RestoreThread(IntPtr tstate) => Delegates.PyEval_RestoreThread(tstate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetBuiltins(); + + internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetGlobals(); + + internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetLocals(); + + internal static IntPtr PyEval_GetLocals() => Delegates.PyEval_GetLocals(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetProgramName(); + + internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetProgramName(IntPtr name); + + internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPythonHome(); + + internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetPythonHome(IntPtr home); + + internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPath(); + + internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetPath(IntPtr home); + + internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetVersion(); + + internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPlatform(); + + internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetCopyright(); + + internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetCompiler(); + + internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetBuildInfo(); + + internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyRun_SimpleString(string code); + + internal static int PyRun_SimpleString(string code) => Delegates.PyRun_SimpleString(code); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); + + internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) => Delegates.PyRun_String(code, st, globals, locals); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); + + internal static IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals) => Delegates.PyEval_EvalCode(co, globals, locals); /// /// Return value: New reference. @@ -952,20 +962,20 @@ internal static IntPtr Py_CompileStringFlags(string str, string file, int start, /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). /// /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); + + internal static IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize) => Delegates.Py_CompileStringExFlags(str, file, start, flags, optimize); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); + + internal static IntPtr PyImport_ExecCodeModule(string name, IntPtr code) => Delegates.PyImport_ExecCodeModule(name, code); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod); + + internal static IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod) => Delegates.PyCFunction_NewEx(ml, self, mod); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); + + internal static IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw) => Delegates.PyCFunction_Call(func, args, kw); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls); + + internal static IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls) => Delegates.PyMethod_New(func, self, cls); //==================================================================== @@ -1026,44 +1036,62 @@ internal static bool PyObject_IsIterable(IntPtr pointer) return tp_iter != IntPtr.Zero; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_HasAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); + + internal static int PyObject_HasAttrString(IntPtr pointer, string name) => Delegates.PyObject_HasAttrString(pointer, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); + internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) + { + IntPtr nameMem = Marshal.StringToHGlobalAnsi(name); + try + { + return Delegates.PyObject_GetAttrString(pointer, nameMem); + } + finally + { + Marshal.FreeHGlobal(nameMem); + } + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name, IntPtr value); + + internal static IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name) => Delegates.PyObject_GetAttrString(pointer, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_HasAttr(IntPtr pointer, IntPtr name); + + internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) => Delegates.PyObject_SetAttrString(pointer, name, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name); + + internal static int PyObject_HasAttr(IntPtr pointer, IntPtr name) => Delegates.PyObject_HasAttr(pointer, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key); + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) + => Delegates.PyObject_GetAttr(pointer, new BorrowedReference(name)); + internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) + => Delegates.PyObject_GetAttr(new BorrowedReference(pointer), new BorrowedReference(name)) + .DangerousMoveToPointerOrNull(); + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_GetAttr(pointer, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_DelItem(IntPtr pointer, IntPtr key); + internal static int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value) => Delegates.PyObject_SetAttr(pointer, name, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetIter(IntPtr op); + + internal static IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_GetItem(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw); + + internal static int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value) => Delegates.PyObject_SetItem(pointer, key, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args); + + internal static int PyObject_DelItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_DelItem(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid); + + internal static IntPtr PyObject_GetIter(IntPtr op) => Delegates.PyObject_GetIter(op); + + + internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); + + + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) => Delegates.PyObject_CallObject(pointer, args); + + + internal static int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); internal static int PyObject_Compare(IntPtr value1, IntPtr value2) { @@ -1090,44 +1118,44 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) return -1; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsInstance(IntPtr ob, IntPtr type); + + internal static int PyObject_IsInstance(IntPtr ob, IntPtr type) => Delegates.PyObject_IsInstance(ob, type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsSubclass(IntPtr ob, IntPtr type); + + internal static int PyObject_IsSubclass(IntPtr ob, IntPtr type) => Delegates.PyObject_IsSubclass(ob, type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCallable_Check(IntPtr pointer); + + internal static int PyCallable_Check(IntPtr pointer) => Delegates.PyCallable_Check(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsTrue(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_Not(IntPtr pointer); + internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); + internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); + + + internal static int PyObject_Not(IntPtr pointer) => Delegates.PyObject_Not(pointer); internal static long PyObject_Size(IntPtr pointer) { return (long)_PyObject_Size(pointer); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); + + private static IntPtr _PyObject_Size(IntPtr pointer) => Delegates._PyObject_Size(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern nint PyObject_Hash(IntPtr op); + + internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Repr(IntPtr pointer); + + internal static IntPtr PyObject_Repr(IntPtr pointer) => Delegates.PyObject_Repr(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Str(IntPtr pointer); + + internal static IntPtr PyObject_Str(IntPtr pointer) => Delegates.PyObject_Str(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyObject_Str")] - internal static extern IntPtr PyObject_Unicode(IntPtr pointer); + + internal static IntPtr PyObject_Unicode(IntPtr pointer) => Delegates.PyObject_Unicode(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Dir(IntPtr pointer); + + internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); #if PYTHON_WITH_PYDEBUG [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] @@ -1138,49 +1166,48 @@ internal static long PyObject_Size(IntPtr pointer) // Python buffer API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags); + + internal static int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, ref view, flags); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyBuffer_Release(ref Py_buffer view); + + internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - internal static extern IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format); + + internal static IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format) => Delegates.PyBuffer_SizeFromFormat(format); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_IsContiguous(ref Py_buffer view, char order); + + internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices); + + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort); + + internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order); + + internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order); + + internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags); + + internal static int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); //==================================================================== // Python number API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyNumber_Long")] - internal static extern IntPtr PyNumber_Int(IntPtr ob); + + internal static IntPtr PyNumber_Int(IntPtr ob) => Delegates.PyNumber_Int(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Long(IntPtr ob); + + internal static IntPtr PyNumber_Long(IntPtr ob) => Delegates.PyNumber_Long(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Float(IntPtr ob); + + internal static IntPtr PyNumber_Float(IntPtr ob) => Delegates.PyNumber_Float(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyNumber_Check(IntPtr ob); + + internal static bool PyNumber_Check(IntPtr ob) => Delegates.PyNumber_Check(ob); internal static bool PyInt_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, new BorrowedReference(PyIntType)); @@ -1206,33 +1233,28 @@ internal static IntPtr PyInt_FromInt64(long value) return PyInt_FromLong(v); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromLong")] - private static extern IntPtr PyInt_FromLong(IntPtr value); + + private static IntPtr PyInt_FromLong(IntPtr value) => Delegates.PyInt_FromLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsLong")] - internal static extern int PyInt_AsLong(IntPtr value); + + internal static int PyInt_AsLong(IntPtr value) => Delegates.PyInt_AsLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromString")] - internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); + + internal static IntPtr PyInt_FromString(string value, IntPtr end, int radix) => Delegates.PyInt_FromString(value, end, radix); internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromLong(long value); + + internal static IntPtr PyLong_FromLong(long value) => Delegates.PyLong_FromLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromUnsignedLong")] - internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + + internal static IntPtr PyLong_FromUnsignedLong32(uint value) => Delegates.PyLong_FromUnsignedLong32(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromUnsignedLong")] - internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong64(ulong value) => Delegates.PyLong_FromUnsignedLong64(value); internal static IntPtr PyLong_FromUnsignedLong(object value) { @@ -1242,27 +1264,25 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromDouble(double value); + + internal static IntPtr PyLong_FromDouble(double value) => Delegates.PyLong_FromDouble(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromLongLong(long value); + + internal static IntPtr PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromUnsignedLongLong(ulong value); + + internal static IntPtr PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromString(string value, IntPtr end, int radix); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsSize_t")] - internal static extern nuint PyLong_AsUnsignedSize_t(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsSsize_t")] - internal static extern nint PyLong_AsSignedSize_t(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsSsize_t")] - internal static extern nint PyLong_AsSignedSize_t(BorrowedReference value); + + internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) => Delegates.PyLong_FromString(value, end, radix); + + + + internal static nuint PyLong_AsUnsignedSize_t(IntPtr value) => Delegates.PyLong_AsUnsignedSize_t(value); + + internal static nint PyLong_AsSignedSize_t(IntPtr value) => Delegates.PyLong_AsSignedSize_t(new BorrowedReference(value)); + + internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); /// /// This function is a rename of PyLong_AsLongLong, which has a commonly undesired @@ -1272,11 +1292,10 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) /// In most cases you need to check that value is an instance of PyLongObject /// before using this function using . /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsLongLong")] - internal static extern long PyExplicitlyConvertToInt64(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern ulong PyLong_AsUnsignedLongLong(IntPtr value); + + internal static long PyExplicitlyConvertToInt64(IntPtr value) => Delegates.PyExplicitlyConvertToInt64(value); + + internal static ulong PyLong_AsUnsignedLongLong(IntPtr value) => Delegates.PyLong_AsUnsignedLongLong(value); internal static bool PyFloat_Check(IntPtr ob) { @@ -1287,199 +1306,193 @@ internal static bool PyFloat_Check(IntPtr ob) /// Return value: New reference. /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + internal static NewReference PyLong_FromVoidPtr(IntPtr p) => Delegates.PyLong_FromVoidPtr(p); /// /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); + + internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyFloat_FromDouble(double value); + + internal static IntPtr PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyFloat_FromString(IntPtr value, IntPtr junk); + + internal static IntPtr PyFloat_FromString(IntPtr value, IntPtr junk) => Delegates.PyFloat_FromString(value, junk); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern double PyFloat_AsDouble(IntPtr ob); + + internal static double PyFloat_AsDouble(IntPtr ob) => Delegates.PyFloat_AsDouble(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Add(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Add(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Add(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Subtract(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Multiply(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_TrueDivide(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_And(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_And(IntPtr o1, IntPtr o2) => Delegates.PyNumber_And(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Xor(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Or(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Or(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Or(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Lshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Rshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Power(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Power(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Power(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Remainder(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceXor(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceOr(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlacePower(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2); + + internal static IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Negative(IntPtr o1); + + internal static IntPtr PyNumber_Negative(IntPtr o1) => Delegates.PyNumber_Negative(o1); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Positive(IntPtr o1); + + internal static IntPtr PyNumber_Positive(IntPtr o1) => Delegates.PyNumber_Positive(o1); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Invert(IntPtr o1); + + internal static IntPtr PyNumber_Invert(IntPtr o1) => Delegates.PyNumber_Invert(o1); //==================================================================== // Python sequence API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PySequence_Check(IntPtr pointer); - - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); + + internal static bool PySequence_Check(IntPtr pointer) => Delegates.PySequence_Check(pointer); + + internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) { return PySequence_SetItem(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + private static int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PySequence_SetItem(pointer, index, value); internal static int PySequence_DelItem(IntPtr pointer, long index) { return PySequence_DelItem(pointer, new IntPtr(index)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); + + private static int PySequence_DelItem(IntPtr pointer, IntPtr index) => Delegates.PySequence_DelItem(pointer, index); internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) { return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + + private static IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) { return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); + + private static int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) { return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + + private static int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); - internal static long PySequence_Size(IntPtr pointer) - { - return (long)_PySequence_Size(pointer); - } + [Obsolete] + internal static nint PySequence_Size(IntPtr pointer) => PySequence_Size(new BorrowedReference(pointer)); + internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); + + internal static int PySequence_Contains(IntPtr pointer, IntPtr item) => Delegates.PySequence_Contains(pointer, item); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); + + internal static IntPtr PySequence_Concat(IntPtr pointer, IntPtr other) => Delegates.PySequence_Concat(pointer, other); internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) { return PySequence_Repeat(pointer, new IntPtr(count)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); + + private static IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count) => Delegates.PySequence_Repeat(pointer, count); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); + + internal static int PySequence_Index(IntPtr pointer, IntPtr item) => Delegates.PySequence_Index(pointer, item); internal static long PySequence_Count(IntPtr pointer, IntPtr value) { return (long)_PySequence_Count(pointer, value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + + private static IntPtr _PySequence_Count(IntPtr pointer, IntPtr value) => Delegates._PySequence_Count(pointer, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_Tuple(IntPtr pointer); + + internal static IntPtr PySequence_Tuple(IntPtr pointer) => Delegates.PySequence_Tuple(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_List(IntPtr pointer); + + internal static IntPtr PySequence_List(IntPtr pointer) => Delegates.PySequence_List(pointer); //==================================================================== // Python string API //==================================================================== - + internal static bool IsStringType(BorrowedReference op) + { + BorrowedReference t = PyObject_TYPE(op); + return (t == new BorrowedReference(PyStringType)) + || (t == new BorrowedReference(PyUnicodeType)); + } internal static bool IsStringType(IntPtr op) { IntPtr t = PyObject_TYPE(op); @@ -1496,16 +1509,16 @@ internal static IntPtr PyString_FromString(string value) return PyUnicode_FromKindAndData(_UCS, value, value.Length); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyBytes_FromString(string op); + + internal static IntPtr PyBytes_FromString(string op) => Delegates.PyBytes_FromString(op); internal static long PyBytes_Size(IntPtr op) { return (long)_PyBytes_Size(op); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); + + private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { @@ -1518,67 +1531,68 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) return PyUnicode_FromStringAndSize(value, new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + + private static IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size) => Delegates.PyUnicode_FromStringAndSize(value, size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); + + internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode); internal static bool PyUnicode_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyUnicodeType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); + + internal static IntPtr PyUnicode_FromObject(IntPtr ob) => Delegates.PyUnicode_FromObject(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); + + internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) { return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( + + private static IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, IntPtr size - ); + ) => Delegates.PyUnicode_FromKindAndData(kind, s, size +); internal static IntPtr PyUnicode_FromUnicode(string s, long size) { return PyUnicode_FromKindAndData(_UCS, s, size); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyUnicode_GetMax(); + + internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax(); internal static long PyUnicode_GetSize(IntPtr ob) { return (long)_PyUnicode_GetSize(ob); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + + private static IntPtr _PyUnicode_GetSize(IntPtr ob) => Delegates._PyUnicode_GetSize(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); + + internal static IntPtr PyUnicode_AsUnicode(IntPtr ob) => Delegates.PyUnicode_AsUnicode(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromOrdinal(int c); + + internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); internal static IntPtr PyUnicode_FromString(string s) { return PyUnicode_FromUnicode(s, s.Length); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_InternFromString(string s); + + internal static IntPtr PyUnicode_InternFromString(string s) => Delegates.PyUnicode_InternFromString(s); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyUnicode_Compare(IntPtr left, IntPtr right); + + internal static int PyUnicode_Compare(IntPtr left, IntPtr right) => Delegates.PyUnicode_Compare(left, right); internal static string GetManagedString(in BorrowedReference borrowedReference) => GetManagedString(borrowedReference.DangerousGetAddress()); @@ -1623,92 +1637,104 @@ internal static bool PyDict_Check(IntPtr ob) return PyObject_TYPE(ob) == PyDictType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_New(); + + internal static IntPtr PyDict_New() => Delegates.PyDict_New(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + + internal static int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue) => Delegates.PyDict_Next(p, out ppos, out pkey, out pvalue); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDictProxy_New(IntPtr dict); + + internal static IntPtr PyDictProxy_New(IntPtr dict) => Delegates.PyDictProxy_New(dict); /// /// Return value: Borrowed reference. - /// Return NULL if the key key is not present, but without setting an exception. + /// Return NULL if the key is not present, but without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key); - + internal static IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key) + => Delegates.PyDict_GetItem(new BorrowedReference(pointer), new BorrowedReference(key)) + .DangerousGetAddressOrNull(); /// - /// Return value: Borrowed reference. + /// Return NULL if the key is not present, but without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); + internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); + + internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) => Delegates.PyDict_GetItemString(pointer, key); + + internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); /// /// Return 0 on success or -1 on failure. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); + [Obsolete] + internal static int PyDict_SetItem(IntPtr dict, IntPtr key, IntPtr value) => Delegates.PyDict_SetItem(new BorrowedReference(dict), new BorrowedReference(key), new BorrowedReference(value)); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, BorrowedReference key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, key, value); /// /// Return 0 on success or -1 on failure. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_SetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key, IntPtr value); + internal static int PyDict_SetItemString(IntPtr dict, string key, IntPtr value) => Delegates.PyDict_SetItemString(new BorrowedReference(dict), key, new BorrowedReference(value)); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) => Delegates.PyDict_SetItemString(dict, key, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_DelItem(IntPtr pointer, IntPtr key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_DelItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); + internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyMapping_HasKey(IntPtr pointer, IntPtr key); + + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) => Delegates.PyDict_DelItemString(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Keys(IntPtr pointer); + + internal static int PyMapping_HasKey(IntPtr pointer, IntPtr key) => Delegates.PyMapping_HasKey(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Values(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyDict_Items(IntPtr pointer); + [Obsolete] + internal static IntPtr PyDict_Keys(IntPtr pointer) + => Delegates.PyDict_Keys(new BorrowedReference(pointer)) + .DangerousMoveToPointerOrNull(); + internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Copy(IntPtr pointer); + + internal static IntPtr PyDict_Values(IntPtr pointer) => Delegates.PyDict_Values(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_Update(IntPtr pointer, IntPtr other); + + internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyDict_Clear(IntPtr pointer); + + internal static IntPtr PyDict_Copy(IntPtr pointer) => Delegates.PyDict_Copy(pointer); + + + internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); + + + internal static void PyDict_Clear(IntPtr pointer) => Delegates.PyDict_Clear(pointer); internal static long PyDict_Size(IntPtr pointer) { return (long)_PyDict_Size(pointer); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + + internal static IntPtr _PyDict_Size(IntPtr pointer) => Delegates._PyDict_Size(pointer); - /// - /// Return value: New reference. - /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySet_New(IntPtr iterable); + internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySet_Add(IntPtr set, IntPtr key); + + internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); /// /// Return 1 if found, 0 if not found, and -1 if an error is encountered. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); + + internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); //==================================================================== // Python list API @@ -1724,73 +1750,72 @@ internal static IntPtr PyList_New(long size) return PyList_New(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); + + private static IntPtr PyList_New(IntPtr size) => Delegates.PyList_New(size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyList_AsTuple(IntPtr pointer); + + internal static IntPtr PyList_AsTuple(IntPtr pointer) => Delegates.PyList_AsTuple(pointer); internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index); + + private static BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyList_GetItem(pointer, index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) { return PyList_SetItem(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + private static int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyList_SetItem(pointer, index, value); internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) { return PyList_Insert(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value); + + private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Append(BorrowedReference pointer, IntPtr value); + + internal static int PyList_Append(BorrowedReference pointer, IntPtr value) => Delegates.PyList_Append(pointer, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Reverse(BorrowedReference pointer); + + internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Sort(BorrowedReference pointer); + + internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) { return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); + + private static IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyList_GetSlice(pointer, start, end); internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) { return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); + + private static int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value) => Delegates.PyList_SetSlice(pointer, start, end, value); - internal static long PyList_Size(BorrowedReference pointer) - { - return (long)_PyList_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(BorrowedReference pointer); + + internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); //==================================================================== // Python tuple API //==================================================================== + internal static bool PyTuple_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == new BorrowedReference(PyTupleType); + } internal static bool PyTuple_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyTupleType; @@ -1801,44 +1826,39 @@ internal static IntPtr PyTuple_New(long size) return PyTuple_New(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); + + private static IntPtr PyTuple_New(IntPtr size) => Delegates.PyTuple_New(size); internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, long index) => PyTuple_GetItem(pointer, new IntPtr(index)); internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) { - return PyTuple_GetItem(pointer, new IntPtr(index)); + return PyTuple_GetItem(new BorrowedReference(pointer), new IntPtr(index)) + .DangerousGetAddressOrNull(); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); + + private static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyTuple_GetItem(pointer, index); internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) { return PyTuple_SetItem(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) { return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); + + private static IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyTuple_GetSlice(pointer, start, end); - internal static long PyTuple_Size(IntPtr pointer) - { - return (long)_PyTuple_Size(pointer); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + internal static nint PyTuple_Size(IntPtr pointer) => PyTuple_Size(new BorrowedReference(pointer)); + internal static nint PyTuple_Size(BorrowedReference pointer) => Delegates.PyTuple_Size(pointer); //==================================================================== @@ -1852,87 +1872,89 @@ internal static bool PyIter_Check(IntPtr pointer) return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyIter_Next(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyIter_Next(BorrowedReference pointer); + + internal static IntPtr PyIter_Next(IntPtr pointer) + => Delegates.PyIter_Next(new BorrowedReference(pointer)).DangerousMoveToPointerOrNull(); + internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); //==================================================================== // Python module API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyModule_New(string name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern string PyModule_GetName(IntPtr module); + internal static NewReference PyModule_New(string name) => Delegates.PyModule_New(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyModule_GetDict(IntPtr module); + + internal static string PyModule_GetName(IntPtr module) => Delegates.PyModule_GetName(module); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern string PyModule_GetFilename(IntPtr module); + + internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); + + + internal static string PyModule_GetFilename(IntPtr module) => Delegates.PyModule_GetFilename(module); #if PYTHON_WITH_PYDEBUG [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] #else - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + #endif - internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); + internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_Import(IntPtr name); + + internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); /// /// Return value: New reference. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ImportModule(string name); + + internal static IntPtr PyImport_ImportModule(string name) => Delegates.PyImport_ImportModule(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ReloadModule(IntPtr module); + + internal static IntPtr PyImport_ReloadModule(IntPtr module) => Delegates.PyImport_ReloadModule(module); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_AddModule(string name); + + internal static BorrowedReference PyImport_AddModule(string name) => Delegates.PyImport_AddModule(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_GetModuleDict(); + + internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PySys_SetArgvEx( + + internal static void PySys_SetArgvEx( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, int updatepath - ); + ) => Delegates.PySys_SetArgvEx(argc, argv, updatepath +); /// /// Return value: Borrowed reference. /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern BorrowedReference PySys_GetObject(string name); + + internal static BorrowedReference PySys_GetObject(string name) => Delegates.PySys_GetObject(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySys_SetObject(string name, IntPtr ob); + + [Obsolete] + internal static int PySys_SetObject(string name, IntPtr ob) => Delegates.PySys_SetObject(name, new BorrowedReference(ob)); + internal static int PySys_SetObject(string name, BorrowedReference ob) => Delegates.PySys_SetObject(name, ob); //==================================================================== // Python type object API //==================================================================== + static readonly delegate* unmanaged[Cdecl] pyType_Check; internal static bool PyType_Check(IntPtr ob) { return PyObject_TypeCheck(ob, PyTypeType); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyType_Modified(IntPtr type); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyType_IsSubtype(IntPtr t1, IntPtr t2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2); + + internal static void PyType_Modified(IntPtr type) => Delegates.PyType_Modified(type); + internal static bool PyType_IsSubtype(BorrowedReference t1, IntPtr ofType) + => PyType_IsSubtype(t1, new BorrowedReference(ofType)); + internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) => Delegates.PyType_IsSubtype(t1, t2); internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) => PyObject_TypeCheck(new BorrowedReference(ob), new BorrowedReference(tp)); @@ -1942,54 +1964,53 @@ internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference return (t == tp) || PyType_IsSubtype(t, tp); } - internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, IntPtr ofType) + => PyType_IsSameAsOrSubtype(type, new BorrowedReference(ofType)); + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedReference ofType) { return (type == ofType) || PyType_IsSubtype(type, ofType); } - /// - /// Generic handler for the tp_new slot of a type object. Create a new instance using the type’s tp_alloc slot. - /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kwds); + + internal static IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw) => Delegates.PyType_GenericNew(type, args, kw); internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) { return PyType_GenericAlloc(type, new IntPtr(n)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + + private static IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n) => Delegates.PyType_GenericAlloc(type, n); /// /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyType_Ready(IntPtr type); + + internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyType_Lookup(IntPtr type, IntPtr name); + + internal static IntPtr _PyType_Lookup(IntPtr type, IntPtr name) => Delegates._PyType_Lookup(type, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name); + + internal static IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name) => Delegates.PyObject_GenericGetAttr(obj, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value); + + internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyObject_GetDictPtr(IntPtr obj); + + internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_Del(IntPtr tp); + + internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_Track(IntPtr tp); + + internal static void PyObject_GC_Track(IntPtr tp) => Delegates.PyObject_GC_Track(tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_UnTrack(IntPtr tp); + + internal static void PyObject_GC_UnTrack(IntPtr tp) => Delegates.PyObject_GC_UnTrack(tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void _PyObject_Dump(IntPtr ob); + + internal static void _PyObject_Dump(IntPtr ob) => Delegates._PyObject_Dump(ob); //==================================================================== // Python memory API @@ -2000,84 +2021,76 @@ internal static IntPtr PyMem_Malloc(long size) return PyMem_Malloc(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); + + private static IntPtr PyMem_Malloc(IntPtr size) => Delegates.PyMem_Malloc(size); internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) { return PyMem_Realloc(ptr, new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); + + private static IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size) => Delegates.PyMem_Realloc(ptr, size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyMem_Free(IntPtr ptr); + + internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); //==================================================================== // Python exception API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetString(IntPtr ob, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string message); + + internal static void PyErr_SetString(IntPtr ob, string message) => Delegates.PyErr_SetString(ob, message); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject); + + internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_SetFromErrno(IntPtr ob); + + internal static IntPtr PyErr_SetFromErrno(IntPtr ob) => Delegates.PyErr_SetFromErrno(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetNone(IntPtr ob); + + internal static void PyErr_SetNone(IntPtr ob) => Delegates.PyErr_SetNone(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyErr_ExceptionMatches(IntPtr exception); + + internal static int PyErr_ExceptionMatches(IntPtr exception) => Delegates.PyErr_ExceptionMatches(exception); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val); + + internal static int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val) => Delegates.PyErr_GivenExceptionMatches(ob, val); - /// - /// Under certain circumstances, the values returned by PyErr_Fetch() below can be “unnormalized”, - /// meaning that *exc is a class object but *val is not an instance of the same class. - /// This function can be used to instantiate the class in that case. - /// If the values are already normalized, nothing happens. - /// The delayed normalization is implemented to improve performance. - /// Must not be called when an error is set. - /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); + + internal static void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb) => Delegates.PyErr_NormalizeException(ref ob, ref val, ref tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); + + internal static IntPtr PyErr_Occurred() => Delegates.PyErr_Occurred(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); + + internal static void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb) => Delegates.PyErr_Fetch(out ob, out val, out tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); + + internal static void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb) => Delegates.PyErr_Restore(ob, val, tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Clear(); + + internal static void PyErr_Clear() => Delegates.PyErr_Clear(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Print(); + + internal static void PyErr_Print() => Delegates.PyErr_Print(); /// /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyException_SetCause(IntPtr ex, IntPtr cause); + + internal static void PyException_SetCause(IntPtr ex, IntPtr cause) => Delegates.PyException_SetCause(ex, cause); //==================================================================== // Cell API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyCell_Get(BorrowedReference cell); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); + internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); + + + internal static int PyCell_Set(BorrowedReference cell, IntPtr value) => Delegates.PyCell_Set(cell, value); //==================================================================== // Python GC API @@ -2089,14 +2102,14 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGC_Collect(); + + internal static IntPtr PyGC_Collect() => Delegates.PyGC_Collect(); - internal static IntPtr _Py_AS_GC(IntPtr ob) + internal static IntPtr _Py_AS_GC(BorrowedReference ob) { // XXX: PyGC_Head has a force alignment depend on platform. // See PyGC_Head in objimpl.h for more details. - return Is32Bit ? ob - 16 : ob - 24; + return ob.DangerousGetAddress() - (Is32Bit ? 16 : 24); } internal static IntPtr _Py_FROM_GC(IntPtr gc) @@ -2118,15 +2131,13 @@ internal static IntPtr _PyGCHead_REFS(IntPtr gc) } } - internal static IntPtr _PyGC_REFS(IntPtr ob) + internal static IntPtr _PyGC_REFS(BorrowedReference ob) { return _PyGCHead_REFS(_Py_AS_GC(ob)); } - internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) - { - return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; - } + internal static bool _PyObject_GC_IS_TRACKED(BorrowedReference ob) + => (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; internal static void Py_CLEAR(ref IntPtr ob) { @@ -2138,36 +2149,35 @@ internal static void Py_CLEAR(ref IntPtr ob) // Python Capsules API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + internal static NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor) => Delegates.PyCapsule_New(pointer, name, destructor); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); + + internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name) => Delegates.PyCapsule_GetPointer(capsule, name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); + + internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); //==================================================================== // Miscellaneous //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_Self(IntPtr ob); + + internal static IntPtr PyMethod_Self(IntPtr ob) => Delegates.PyMethod_Self(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_Function(IntPtr ob); + + internal static IntPtr PyMethod_Function(IntPtr ob) => Delegates.PyMethod_Function(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + internal static int Py_AddPendingCall(IntPtr func, IntPtr arg) => Delegates.Py_AddPendingCall(func, arg); - [DllImport(_PythonDll, EntryPoint = "PyThreadState_SetAsyncExc", CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThreadState_SetAsyncExcLLP64(uint id, IntPtr exc); + + internal static int PyThreadState_SetAsyncExcLLP64(uint id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); + + internal static int PyThreadState_SetAsyncExcLP64(ulong id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); - [DllImport(_PythonDll, EntryPoint = "PyThreadState_SetAsyncExc", CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThreadState_SetAsyncExcLP64(ulong id, IntPtr exc); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_MakePendingCalls(); + internal static int Py_MakePendingCalls() => Delegates.Py_MakePendingCalls(); internal static void SetNoSiteFlag() { @@ -2202,6 +2212,532 @@ internal static IntPtr GetBuiltins() { return PyImport_Import(PyIdentifier.builtins); } + + private static class Delegates + { + static Delegates() + { + PyDictProxy_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDictProxy_New), GetUnmanagedDll(_PythonDll)); +Py_IncRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IncRef), GetUnmanagedDll(_PythonDll)); + Py_DecRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_DecRef), GetUnmanagedDll(_PythonDll)); + Py_Initialize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Initialize), GetUnmanagedDll(_PythonDll)); + Py_InitializeEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_InitializeEx), GetUnmanagedDll(_PythonDll)); + Py_IsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IsInitialized), GetUnmanagedDll(_PythonDll)); + Py_Finalize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Finalize), GetUnmanagedDll(_PythonDll)); + Py_NewInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_NewInterpreter), GetUnmanagedDll(_PythonDll)); + Py_EndInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_EndInterpreter), GetUnmanagedDll(_PythonDll)); + PyThreadState_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_New), GetUnmanagedDll(_PythonDll)); + PyThreadState_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Get), GetUnmanagedDll(_PythonDll)); + _PyThreadState_UncheckedGet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyThreadState_UncheckedGet), GetUnmanagedDll(_PythonDll)); + PyThread_get_key_value = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_get_key_value), GetUnmanagedDll(_PythonDll)); + PyThread_get_thread_ident = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_get_thread_ident), GetUnmanagedDll(_PythonDll)); + PyThread_set_key_value = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_set_key_value), GetUnmanagedDll(_PythonDll)); + PyThreadState_Swap = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Swap), GetUnmanagedDll(_PythonDll)); + PyGILState_Ensure = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Ensure), GetUnmanagedDll(_PythonDll)); + PyGILState_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Release), GetUnmanagedDll(_PythonDll)); + PyGILState_GetThisThreadState = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_GetThisThreadState), GetUnmanagedDll(_PythonDll)); + Py_Main = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Main), GetUnmanagedDll(_PythonDll)); + PyEval_InitThreads = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_InitThreads), GetUnmanagedDll(_PythonDll)); + PyEval_ThreadsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ThreadsInitialized), GetUnmanagedDll(_PythonDll)); + PyEval_AcquireLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireLock), GetUnmanagedDll(_PythonDll)); + PyEval_ReleaseLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ReleaseLock), GetUnmanagedDll(_PythonDll)); + PyEval_AcquireThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireThread), GetUnmanagedDll(_PythonDll)); + PyEval_ReleaseThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ReleaseThread), GetUnmanagedDll(_PythonDll)); + PyEval_SaveThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_SaveThread), GetUnmanagedDll(_PythonDll)); + PyEval_RestoreThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_RestoreThread), GetUnmanagedDll(_PythonDll)); + PyEval_GetBuiltins = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetBuiltins), GetUnmanagedDll(_PythonDll)); + PyEval_GetGlobals = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetGlobals), GetUnmanagedDll(_PythonDll)); + PyEval_GetLocals = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetLocals), GetUnmanagedDll(_PythonDll)); + Py_GetProgramName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetProgramName), GetUnmanagedDll(_PythonDll)); + Py_SetProgramName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetProgramName), GetUnmanagedDll(_PythonDll)); + Py_GetPythonHome = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPythonHome), GetUnmanagedDll(_PythonDll)); + Py_SetPythonHome = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetPythonHome), GetUnmanagedDll(_PythonDll)); + Py_GetPath = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPath), GetUnmanagedDll(_PythonDll)); + Py_SetPath = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetPath), GetUnmanagedDll(_PythonDll)); + Py_GetVersion = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetVersion), GetUnmanagedDll(_PythonDll)); + Py_GetPlatform = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPlatform), GetUnmanagedDll(_PythonDll)); + Py_GetCopyright = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCopyright), GetUnmanagedDll(_PythonDll)); + Py_GetCompiler = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCompiler), GetUnmanagedDll(_PythonDll)); + Py_GetBuildInfo = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetBuildInfo), GetUnmanagedDll(_PythonDll)); + PyRun_SimpleString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_SimpleString), GetUnmanagedDll(_PythonDll)); + PyRun_String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_String), GetUnmanagedDll(_PythonDll)); + PyEval_EvalCode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_EvalCode), GetUnmanagedDll(_PythonDll)); + Py_CompileStringExFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringExFlags), GetUnmanagedDll(_PythonDll)); + PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); + PyCFunction_NewEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_NewEx), GetUnmanagedDll(_PythonDll)); + PyCFunction_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_Call), GetUnmanagedDll(_PythonDll)); + PyMethod_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_New), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_SetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetItem), GetUnmanagedDll(_PythonDll)); + PyObject_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetItem), GetUnmanagedDll(_PythonDll)); + PyObject_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_DelItem), GetUnmanagedDll(_PythonDll)); + PyObject_GetIter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetIter), GetUnmanagedDll(_PythonDll)); + PyObject_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Call), GetUnmanagedDll(_PythonDll)); + PyObject_CallObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_CallObject), GetUnmanagedDll(_PythonDll)); + PyObject_RichCompareBool = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_RichCompareBool), GetUnmanagedDll(_PythonDll)); + PyObject_IsInstance = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsInstance), GetUnmanagedDll(_PythonDll)); + PyObject_IsSubclass = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsSubclass), GetUnmanagedDll(_PythonDll)); + PyCallable_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCallable_Check), GetUnmanagedDll(_PythonDll)); + PyObject_IsTrue = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsTrue), GetUnmanagedDll(_PythonDll)); + PyObject_Not = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Not), GetUnmanagedDll(_PythonDll)); + _PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); + PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); + PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); + PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); + PyObject_Unicode = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Str", GetUnmanagedDll(_PythonDll)); + PyObject_Dir = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll)); + PyObject_GetBuffer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll)); + PyBuffer_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll)); + PyBuffer_SizeFromFormat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_SizeFromFormat), GetUnmanagedDll(_PythonDll)); + PyBuffer_IsContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_IsContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll)); + PyBuffer_FromContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FromContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_ToContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_ToContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_FillContiguousStrides = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FillContiguousStrides), GetUnmanagedDll(_PythonDll)); + PyBuffer_FillInfo = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FillInfo), GetUnmanagedDll(_PythonDll)); + PyNumber_Int = (delegate* unmanaged[Cdecl])GetFunctionByName("PyNumber_Long", GetUnmanagedDll(_PythonDll)); + PyNumber_Long = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Long), GetUnmanagedDll(_PythonDll)); + PyNumber_Float = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Float), GetUnmanagedDll(_PythonDll)); + PyNumber_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Check), GetUnmanagedDll(_PythonDll)); + PyInt_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromLong", GetUnmanagedDll(_PythonDll)); + PyInt_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLong", GetUnmanagedDll(_PythonDll)); + PyInt_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromString", GetUnmanagedDll(_PythonDll)); + PyLong_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromDouble), GetUnmanagedDll(_PythonDll)); + PyLong_FromLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromUnsignedLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromString), GetUnmanagedDll(_PythonDll)); + PyLong_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsLong), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_AsLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsUnsignedLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromVoidPtr), GetUnmanagedDll(_PythonDll)); + PyLong_AsVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsVoidPtr), GetUnmanagedDll(_PythonDll)); + PyFloat_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromDouble), GetUnmanagedDll(_PythonDll)); + PyFloat_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromString), GetUnmanagedDll(_PythonDll)); + PyFloat_AsDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_AsDouble), GetUnmanagedDll(_PythonDll)); + PyNumber_Add = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Add), GetUnmanagedDll(_PythonDll)); + PyNumber_Subtract = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Subtract), GetUnmanagedDll(_PythonDll)); + PyNumber_Multiply = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Multiply), GetUnmanagedDll(_PythonDll)); + PyNumber_TrueDivide = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_TrueDivide), GetUnmanagedDll(_PythonDll)); + PyNumber_And = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_And), GetUnmanagedDll(_PythonDll)); + PyNumber_Xor = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Xor), GetUnmanagedDll(_PythonDll)); + PyNumber_Or = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Or), GetUnmanagedDll(_PythonDll)); + PyNumber_Lshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Lshift), GetUnmanagedDll(_PythonDll)); + PyNumber_Rshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Rshift), GetUnmanagedDll(_PythonDll)); + PyNumber_Power = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Power), GetUnmanagedDll(_PythonDll)); + PyNumber_Remainder = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Remainder), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceAdd = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceAdd), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceSubtract = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceSubtract), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceMultiply = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceMultiply), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceTrueDivide = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceTrueDivide), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceAnd = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceAnd), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceXor = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceXor), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceOr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceOr), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceLshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceLshift), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceRshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceRshift), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlacePower = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlacePower), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceRemainder = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceRemainder), GetUnmanagedDll(_PythonDll)); + PyNumber_Negative = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Negative), GetUnmanagedDll(_PythonDll)); + PyNumber_Positive = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Positive), GetUnmanagedDll(_PythonDll)); + PyNumber_Invert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Invert), GetUnmanagedDll(_PythonDll)); + PySequence_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Check), GetUnmanagedDll(_PythonDll)); + PySequence_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_GetItem), GetUnmanagedDll(_PythonDll)); + PySequence_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_SetItem), GetUnmanagedDll(_PythonDll)); + PySequence_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_DelItem), GetUnmanagedDll(_PythonDll)); + PySequence_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_GetSlice), GetUnmanagedDll(_PythonDll)); + PySequence_SetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_SetSlice), GetUnmanagedDll(_PythonDll)); + PySequence_DelSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_DelSlice), GetUnmanagedDll(_PythonDll)); + PySequence_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Size", GetUnmanagedDll(_PythonDll)); + PySequence_Contains = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Contains), GetUnmanagedDll(_PythonDll)); + PySequence_Concat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Concat), GetUnmanagedDll(_PythonDll)); + PySequence_Repeat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Repeat), GetUnmanagedDll(_PythonDll)); + PySequence_Index = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Index), GetUnmanagedDll(_PythonDll)); + _PySequence_Count = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Count", GetUnmanagedDll(_PythonDll)); + PySequence_Tuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Tuple), GetUnmanagedDll(_PythonDll)); + PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); + PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); + _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); + PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll)); + PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); + _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromOrdinal = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromOrdinal), GetUnmanagedDll(_PythonDll)); + PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); + PyUnicode_Compare = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_Compare), GetUnmanagedDll(_PythonDll)); + PyDict_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_New), GetUnmanagedDll(_PythonDll)); + PyDict_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Next), GetUnmanagedDll(_PythonDll)); + PyDict_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItem), GetUnmanagedDll(_PythonDll)); + PyDict_GetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItem), GetUnmanagedDll(_PythonDll)); + PyDict_SetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItem), GetUnmanagedDll(_PythonDll)); + PyDict_DelItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItemString), GetUnmanagedDll(_PythonDll)); + PyMapping_HasKey = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMapping_HasKey), GetUnmanagedDll(_PythonDll)); + PyDict_Keys = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Keys), GetUnmanagedDll(_PythonDll)); + PyDict_Values = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Values), GetUnmanagedDll(_PythonDll)); + PyDict_Items = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Items), GetUnmanagedDll(_PythonDll)); + PyDict_Copy = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Copy), GetUnmanagedDll(_PythonDll)); + PyDict_Update = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Update), GetUnmanagedDll(_PythonDll)); + PyDict_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Clear), GetUnmanagedDll(_PythonDll)); + _PyDict_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyDict_Size", GetUnmanagedDll(_PythonDll)); + PySet_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_New), GetUnmanagedDll(_PythonDll)); + PySet_Add = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_Add), GetUnmanagedDll(_PythonDll)); + PySet_Contains = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_Contains), GetUnmanagedDll(_PythonDll)); + PyList_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_New), GetUnmanagedDll(_PythonDll)); + PyList_AsTuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_AsTuple), GetUnmanagedDll(_PythonDll)); + PyList_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetItem), GetUnmanagedDll(_PythonDll)); + PyList_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetItem), GetUnmanagedDll(_PythonDll)); + PyList_Insert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Insert), GetUnmanagedDll(_PythonDll)); + PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); + PyList_Reverse = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Reverse), GetUnmanagedDll(_PythonDll)); + PyList_Sort = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Sort), GetUnmanagedDll(_PythonDll)); + PyList_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetSlice), GetUnmanagedDll(_PythonDll)); + PyList_SetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetSlice), GetUnmanagedDll(_PythonDll)); + PyList_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Size), GetUnmanagedDll(_PythonDll)); + PyTuple_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_New), GetUnmanagedDll(_PythonDll)); + PyTuple_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_GetItem), GetUnmanagedDll(_PythonDll)); + PyTuple_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_SetItem), GetUnmanagedDll(_PythonDll)); + PyTuple_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_GetSlice), GetUnmanagedDll(_PythonDll)); + PyTuple_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_Size), GetUnmanagedDll(_PythonDll)); + PyIter_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyIter_Next), GetUnmanagedDll(_PythonDll)); + PyModule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_New), GetUnmanagedDll(_PythonDll)); + PyModule_GetName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll)); + PyModule_GetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll)); + PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); + PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); + PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); + PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); + PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); + PyImport_AddModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_AddModule), GetUnmanagedDll(_PythonDll)); + PyImport_GetModuleDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_GetModuleDict), GetUnmanagedDll(_PythonDll)); + PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); + PySys_GetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_GetObject), GetUnmanagedDll(_PythonDll)); + PySys_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetObject), GetUnmanagedDll(_PythonDll)); + PyType_Modified = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Modified), GetUnmanagedDll(_PythonDll)); + PyType_IsSubtype = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_IsSubtype), GetUnmanagedDll(_PythonDll)); + PyType_GenericNew = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericNew), GetUnmanagedDll(_PythonDll)); + PyType_GenericAlloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericAlloc), GetUnmanagedDll(_PythonDll)); + PyType_Ready = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Ready), GetUnmanagedDll(_PythonDll)); + _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); + PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); + _PyObject_GetDictPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GetDictPtr), GetUnmanagedDll(_PythonDll)); + PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); + PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); + PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); + _PyObject_Dump = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_Dump), GetUnmanagedDll(_PythonDll)); + PyMem_Malloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Malloc), GetUnmanagedDll(_PythonDll)); + PyMem_Realloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Realloc), GetUnmanagedDll(_PythonDll)); + PyMem_Free = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Free), GetUnmanagedDll(_PythonDll)); + PyErr_SetString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetString), GetUnmanagedDll(_PythonDll)); + PyErr_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetObject), GetUnmanagedDll(_PythonDll)); + PyErr_SetFromErrno = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetFromErrno), GetUnmanagedDll(_PythonDll)); + PyErr_SetNone = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetNone), GetUnmanagedDll(_PythonDll)); + PyErr_ExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_ExceptionMatches), GetUnmanagedDll(_PythonDll)); + PyErr_GivenExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_GivenExceptionMatches), GetUnmanagedDll(_PythonDll)); + PyErr_NormalizeException = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_NormalizeException), GetUnmanagedDll(_PythonDll)); + PyErr_Occurred = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Occurred), GetUnmanagedDll(_PythonDll)); + PyErr_Fetch = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Fetch), GetUnmanagedDll(_PythonDll)); + PyErr_Restore = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Restore), GetUnmanagedDll(_PythonDll)); + PyErr_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Clear), GetUnmanagedDll(_PythonDll)); + PyErr_Print = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Print), GetUnmanagedDll(_PythonDll)); + PyCell_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Get), GetUnmanagedDll(_PythonDll)); + PyCell_Set = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Set), GetUnmanagedDll(_PythonDll)); + PyGC_Collect = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGC_Collect), GetUnmanagedDll(_PythonDll)); + PyCapsule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_New), GetUnmanagedDll(_PythonDll)); + PyCapsule_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_GetPointer), GetUnmanagedDll(_PythonDll)); + PyCapsule_SetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_SetPointer), GetUnmanagedDll(_PythonDll)); + PyMethod_Self = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Self), GetUnmanagedDll(_PythonDll)); + PyMethod_Function = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Function), GetUnmanagedDll(_PythonDll)); + Py_AddPendingCall = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_AddPendingCall), GetUnmanagedDll(_PythonDll)); + Py_MakePendingCalls = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_MakePendingCalls), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedSize_t = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsSize_t", GetUnmanagedDll(_PythonDll)); + PyLong_AsSignedSize_t = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsSsize_t", GetUnmanagedDll(_PythonDll)); + PyExplicitlyConvertToInt64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLongLong", GetUnmanagedDll(_PythonDll)); + PyDict_GetItemWithError = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemWithError), GetUnmanagedDll(_PythonDll)); + PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); + PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + } + + static global::System.IntPtr GetUnmanagedDll(string libraryName) + { + throw new NotImplementedException(); + } + + static global::System.IntPtr GetFunctionByName(string functionName, global::System.IntPtr libraryHandle) + { + throw new NotImplementedException(); + } + + internal static delegate* unmanaged[Cdecl] PyDictProxy_New { get; } + internal static delegate* unmanaged[Cdecl] Py_IncRef { get; } + internal static delegate* unmanaged[Cdecl] Py_DecRef { get; } + internal static delegate* unmanaged[Cdecl] Py_Initialize { get; } + internal static delegate* unmanaged[Cdecl] Py_InitializeEx { get; } + internal static delegate* unmanaged[Cdecl] Py_IsInitialized { get; } + internal static delegate* unmanaged[Cdecl] Py_Finalize { get; } + internal static delegate* unmanaged[Cdecl] Py_NewInterpreter { get; } + internal static delegate* unmanaged[Cdecl] Py_EndInterpreter { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_New { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_Get { get; } + internal static delegate* unmanaged[Cdecl] _PyThreadState_UncheckedGet { get; } + internal static delegate* unmanaged[Cdecl] PyThread_get_key_value { get; } + internal static delegate* unmanaged[Cdecl] PyThread_get_thread_ident { get; } + internal static delegate* unmanaged[Cdecl] PyThread_set_key_value { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_Swap { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_Ensure { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_Release { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_GetThisThreadState { get; } + internal static delegate* unmanaged[Cdecl] Py_Main { get; } + internal static delegate* unmanaged[Cdecl] PyEval_InitThreads { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ThreadsInitialized { get; } + internal static delegate* unmanaged[Cdecl] PyEval_AcquireLock { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ReleaseLock { get; } + internal static delegate* unmanaged[Cdecl] PyEval_AcquireThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ReleaseThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_SaveThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_RestoreThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetBuiltins { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetGlobals { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetLocals { get; } + internal static delegate* unmanaged[Cdecl] Py_GetProgramName { get; } + internal static delegate* unmanaged[Cdecl] Py_SetProgramName { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPythonHome { get; } + internal static delegate* unmanaged[Cdecl] Py_SetPythonHome { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPath { get; } + internal static delegate* unmanaged[Cdecl] Py_SetPath { get; } + internal static delegate* unmanaged[Cdecl] Py_GetVersion { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPlatform { get; } + internal static delegate* unmanaged[Cdecl] Py_GetCopyright { get; } + internal static delegate* unmanaged[Cdecl] Py_GetCompiler { get; } + internal static delegate* unmanaged[Cdecl] Py_GetBuildInfo { get; } + internal static delegate* unmanaged[Cdecl] PyRun_SimpleString { get; } + internal static delegate* unmanaged[Cdecl] PyRun_String { get; } + internal static delegate* unmanaged[Cdecl] PyEval_EvalCode { get; } + internal static delegate* unmanaged[Cdecl] Py_CompileStringExFlags { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } + internal static delegate* unmanaged[Cdecl] PyCFunction_NewEx { get; } + internal static delegate* unmanaged[Cdecl] PyCFunction_Call { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_New { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetIter { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Call { get; } + internal static delegate* unmanaged[Cdecl] PyObject_CallObject { get; } + internal static delegate* unmanaged[Cdecl] PyObject_RichCompareBool { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsInstance { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsSubclass { get; } + internal static delegate* unmanaged[Cdecl] PyCallable_Check { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsTrue { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Not { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_Size { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Unicode { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Dir { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetBuffer { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_SizeFromFormat { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_IsContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FromContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_ToContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FillContiguousStrides { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FillInfo { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Int { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Long { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Float { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Check { get; } + internal static delegate* unmanaged[Cdecl] PyInt_FromLong { get; } + internal static delegate* unmanaged[Cdecl] PyInt_AsLong { get; } + internal static delegate* unmanaged[Cdecl] PyInt_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong32 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong64 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromDouble { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong32 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong64 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromVoidPtr { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsVoidPtr { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_FromDouble { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_AsDouble { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Add { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Subtract { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Multiply { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_TrueDivide { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_And { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Xor { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Or { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Lshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Rshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Power { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Remainder { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceAdd { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceSubtract { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceMultiply { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceTrueDivide { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceAnd { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceXor { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceOr { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceLshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceRshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlacePower { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceRemainder { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Negative { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Positive { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Invert { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Check { get; } + internal static delegate* unmanaged[Cdecl] PySequence_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_SetSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_DelSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Size { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Contains { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Concat { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Repeat { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Index { get; } + internal static delegate* unmanaged[Cdecl] _PySequence_Count { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Tuple { get; } + internal static delegate* unmanaged[Cdecl] PySequence_List { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } + internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromStringAndSize { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromKindAndData { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } + internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromOrdinal { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_Compare { get; } + internal static delegate* unmanaged[Cdecl] PyDict_New { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Next { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_SetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_DelItemString { get; } + internal static delegate* unmanaged[Cdecl] PyMapping_HasKey { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Keys { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Values { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Items { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Copy { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Update { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Clear { get; } + internal static delegate* unmanaged[Cdecl] _PyDict_Size { get; } + internal static delegate* unmanaged[Cdecl] PySet_New { get; } + internal static delegate* unmanaged[Cdecl] PySet_Add { get; } + internal static delegate* unmanaged[Cdecl] PySet_Contains { get; } + internal static delegate* unmanaged[Cdecl] PyList_New { get; } + internal static delegate* unmanaged[Cdecl] PyList_AsTuple { get; } + internal static delegate* unmanaged[Cdecl] PyList_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyList_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyList_Insert { get; } + internal static delegate* unmanaged[Cdecl] PyList_Append { get; } + internal static delegate* unmanaged[Cdecl] PyList_Reverse { get; } + internal static delegate* unmanaged[Cdecl] PyList_Sort { get; } + internal static delegate* unmanaged[Cdecl] PyList_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyList_SetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyList_Size { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_New { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_Size { get; } + internal static delegate* unmanaged[Cdecl] PyIter_Next { get; } + internal static delegate* unmanaged[Cdecl] PyModule_New { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetName { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } + internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } + internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_AddModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_GetModuleDict { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } + internal static delegate* unmanaged[Cdecl] PySys_GetObject { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetObject { get; } + internal static delegate* unmanaged[Cdecl] PyType_Modified { get; } + internal static delegate* unmanaged[Cdecl] PyType_IsSubtype { get; } + internal static delegate* unmanaged[Cdecl] PyType_GenericNew { get; } + internal static delegate* unmanaged[Cdecl] PyType_GenericAlloc { get; } + internal static delegate* unmanaged[Cdecl] PyType_Ready { get; } + internal static delegate* unmanaged[Cdecl] _PyType_Lookup { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_GetDictPtr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_Dump { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Malloc { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Realloc { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Free { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetString { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetObject { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetFromErrno { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetNone { get; } + internal static delegate* unmanaged[Cdecl] PyErr_ExceptionMatches { get; } + internal static delegate* unmanaged[Cdecl] PyErr_GivenExceptionMatches { get; } + internal static delegate* unmanaged[Cdecl] PyErr_NormalizeException { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Occurred { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Fetch { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Restore { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Clear { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Print { get; } + internal static delegate* unmanaged[Cdecl] PyCell_Get { get; } + internal static delegate* unmanaged[Cdecl] PyCell_Set { get; } + internal static delegate* unmanaged[Cdecl] PyGC_Collect { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_New { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_SetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_Self { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_Function { get; } + internal static delegate* unmanaged[Cdecl] Py_AddPendingCall { get; } + internal static delegate* unmanaged[Cdecl] Py_MakePendingCalls { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedSize_t { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsSignedSize_t { get; } + internal static delegate* unmanaged[Cdecl] PyExplicitlyConvertToInt64 { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItemWithError { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } + } } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index f45e76db4..1d1cd3ad0 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -295,7 +295,9 @@ private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) var pyMoudles = PyImport_GetModuleDict(); foreach (var item in modules) { - int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + var moduleName = new BorrowedReference(item.Key); + var module = new BorrowedReference(item.Value); + int res = PyDict_SetItem(pyMoudles, moduleName, module); PythonException.ThrowIfIsNotZero(res); XDecref(item.Key); XDecref(item.Value); diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 69acbcd31..295219675 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -19,20 +19,20 @@ public static void Save() throw new Exception("Runtime State set already"); } - IntPtr objs = IntPtr.Zero; + NewReference objs = default; if (ShouldRestoreObjects) { - objs = PySet_New(IntPtr.Zero); - foreach (var obj in PyGCGetObjects()) + objs = PySet_New(default); + foreach (var objRaw in PyGCGetObjects()) { - AddObjPtrToSet(objs, obj); + AddObjPtrToSet(objs, new BorrowedReference(objRaw)); } } - var modules = PySet_New(IntPtr.Zero); + var modules = PySet_New(default); foreach (var name in GetModuleNames()) { - int res = PySet_Add(modules, name); + int res = PySet_Add(modules, new BorrowedReference(name)); PythonException.ThrowIfIsNotZero(res); } @@ -46,10 +46,9 @@ public static void Save() head->gc.gc_refs = IntPtr.Zero; } { - var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + using var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); int res = PySys_SetObject("dummy_gc", pyDummyGC); PythonException.ThrowIfIsNotZero(res); - XDecref(pyDummyGC); try { @@ -58,7 +57,7 @@ public static void Save() } finally { - XDecref(modules); + modules.Dispose(); } if (ShouldRestoreObjects) @@ -71,7 +70,7 @@ public static void Save() } finally { - XDecref(objs); + objs.Dispose(); } } } @@ -79,8 +78,8 @@ public static void Save() public static void Restore() { - var dummyGCAddr = PySys_GetObject("dummy_gc").DangerousGetAddress(); - if (dummyGCAddr == IntPtr.Zero) + var dummyGCAddr = PySys_GetObject("dummy_gc"); + if (dummyGCAddr.IsNull) { throw new InvalidOperationException("Runtime state have not set"); } @@ -97,9 +96,10 @@ private static void ResotreModules(IntPtr dummyGC) var intialModules = PySys_GetObject("initial_modules"); Debug.Assert(!intialModules.IsNull); var modules = PyImport_GetModuleDict(); - foreach (var name in GetModuleNames()) + foreach (var nameRaw in GetModuleNames()) { - if (PySet_Contains(intialModules.DangerousGetAddress(), name) == 1) + var name = new BorrowedReference(nameRaw); + if (PySet_Contains(intialModules, name) == 1) { continue; } @@ -122,21 +122,15 @@ private static void RestoreObjects(IntPtr dummyGC) { throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); } - IntPtr intialObjs = PySys_GetObject("initial_objs").DangerousGetAddress(); - Debug.Assert(intialObjs != IntPtr.Zero); - foreach (var obj in PyGCGetObjects()) + BorrowedReference intialObjs = PySys_GetObject("initial_objs"); + Debug.Assert(@intialObjs.IsNull); + foreach (var objRaw in PyGCGetObjects()) { - var p = PyLong_FromVoidPtr(obj); - try - { - if (PySet_Contains(intialObjs, p) == 1) - { - continue; - } - } - finally + using var p = PyLong_FromVoidPtr(objRaw); + var obj = new BorrowedReference(objRaw); + if (PySet_Contains(intialObjs, p) == 1) { - XDecref(p); + continue; } Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); ExchangeGCChain(obj, dummyGC); @@ -162,34 +156,28 @@ public static IEnumerable PyGCGetObjects() public static IEnumerable GetModuleNames() { var modules = PyImport_GetModuleDict(); - var names = PyDict_Keys(modules); - var length = PyList_Size(new BorrowedReference(names)); + using var names = PyDict_Keys(modules); + var length = PyList_Size(names); + var result = new IntPtr[length]; for (int i = 0; i < length; i++) { - var name = PyList_GetItem(new BorrowedReference(names), i); - yield return name.DangerousGetAddress(); + result[i] = PyList_GetItem(names, i).DangerousGetAddress(); } - XDecref(names); + return result; } - private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + private static void AddObjPtrToSet(BorrowedReference set, BorrowedReference obj) { - var p = PyLong_FromVoidPtr(obj); - XIncref(obj); - try - { - int res = PySet_Add(set, p); - PythonException.ThrowIfIsNotZero(res); - } - finally - { - XDecref(p); - } + IntPtr objRaw = obj.DangerousGetAddress(); + using var p = PyLong_FromVoidPtr(objRaw); + XIncref(objRaw); + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); } /// /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. /// - private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + private static void ExchangeGCChain(BorrowedReference obj, IntPtr gc) { var head = _Py_AS_GC(obj); if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3e9e44a46..3a107d53d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -102,6 +102,7 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) /// object. These Python type instances are used to implement internal /// descriptor and utility types like ModuleObject, PropertyObject, etc. /// + [Obsolete] internal static IntPtr GetTypeHandle(Type type) { // Note that these types are cached with a refcount of 1, so they @@ -117,6 +118,14 @@ internal static IntPtr GetTypeHandle(Type type) _slotsImpls.Add(type, type); return handle; } + /// + /// Given a managed Type derived from ExtensionType, get the handle to + /// a Python type object that delegates its implementation to the Type + /// object. These Python type instances are used to implement internal + /// descriptor and utility types like ModuleObject, PropertyObject, etc. + /// + internal static BorrowedReference GetTypeReference(Type type) + => new BorrowedReference(GetTypeHandle(type)); /// @@ -171,10 +180,10 @@ internal static IntPtr CreateType(Type impl) throw new PythonException(); } - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); + var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); + var mod = NewReference.DangerousFromPointer(Runtime.PyString_FromString("CLR")); Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); - Runtime.XDecref(mod); + mod.Dispose(); InitMethods(type, impl); @@ -294,11 +303,11 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) throw new PythonException(); } - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); string mn = clrType.Namespace ?? ""; - IntPtr mod = Runtime.PyString_FromString(mn); + var mod = NewReference.DangerousFromPointer(Runtime.PyString_FromString(mn)); Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); - Runtime.XDecref(mod); + mod.Dispose(); // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); @@ -372,11 +381,11 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // by default the class dict will have all the C# methods in it, but as this is a // derived class we want the python overrides in there instead if they exist. - IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); - ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, py_dict)); + var cls_dict = new BorrowedReference(Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict)); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, new BorrowedReference(py_dict))); Runtime.XIncref(py_type); // Update the __classcell__ if it exists - var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); + BorrowedReference cell = Runtime.PyDict_GetItemString(cls_dict, "__classcell__"); if (!cell.IsNull) { ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); @@ -522,7 +531,7 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => { - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + var tp_dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) { Runtime.PyErr_Print(); @@ -913,32 +922,23 @@ static class SlotHelper { public static IntPtr CreateObjectType() { - IntPtr globals = Runtime.PyDict_New(); + using var globals = NewReference.DangerousFromPointer(Runtime.PyDict_New()); if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) { - Runtime.XDecref(globals); + globals.Dispose(); throw new PythonException(); } const string code = "class A(object): pass"; - var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); - IntPtr res = resRef.DangerousGetAddress(); - if (res == IntPtr.Zero) + using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + if (resRef.IsNull()) { - try - { - throw new PythonException(); - } - finally - { - Runtime.XDecref(globals); - } + globals.Dispose(); + throw new PythonException(); } resRef.Dispose(); - IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); - Debug.Assert(A != IntPtr.Zero); - Runtime.XIncref(A); - Runtime.XDecref(globals); - return A; + BorrowedReference A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(!A.IsNull); + return Runtime.NewRef(A).DangerousMoveToPointer(); } } } From 51e5184f8899077a960a283a695aecdc8854a12c Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 10 Dec 2020 13:48:52 -0800 Subject: [PATCH 024/151] allow setting PythonDLL --- src/runtime/runtime.cs | 61 +++++++++++------------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 889af1565..f51a18e46 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -22,46 +22,18 @@ public unsafe class Runtime public static int UCS => _UCS; internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; -#if PYTHON36 - const string _minor = "6"; -#elif PYTHON37 - const string _minor = "7"; -#elif PYTHON38 - const string _minor = "8"; -#elif PYTHON39 - const string _minor = "9"; -#else -#error You must define one of PYTHON36 to PYTHON39 -#endif - -#if WINDOWS - internal const string dllBase = "python3" + _minor; -#else - internal const string dllBase = "python3." + _minor; -#endif - -#if PYTHON_WITH_PYDEBUG - internal const string dllWithPyDebug = "d"; -#else - internal const string dllWithPyDebug = ""; -#endif -#if PYTHON_WITH_PYMALLOC - internal const string dllWithPyMalloc = "m"; -#else - internal const string dllWithPyMalloc = ""; -#endif - - // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow - // binary substitution of different Python.Runtime.dll builds in a target application. - - public static readonly string PythonDLL = _PythonDll; + public static string PythonDLL + { + get => _PythonDll; + set + { + if (_isInitialized) + throw new InvalidOperationException("This property must be set before runtime is initialized"); + _PythonDll = value; + } + } -#if PYTHON_WITHOUT_ENABLE_SHARED && !NETSTANDARD - internal const string _PythonDll = "__Internal"; -#else - internal const string _PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; -#endif + static string _PythonDll; // set to true when python is finalizing internal static object IsFinalizingLock = new object(); @@ -2215,6 +2187,8 @@ internal static IntPtr GetBuiltins() private static class Delegates { + static readonly ILibraryLoader libraryLoader = LibraryLoader.Get(); + static Delegates() { PyDictProxy_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDictProxy_New), GetUnmanagedDll(_PythonDll)); @@ -2473,15 +2447,10 @@ static Delegates() PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); } - static global::System.IntPtr GetUnmanagedDll(string libraryName) - { - throw new NotImplementedException(); - } + static global::System.IntPtr GetUnmanagedDll(string libraryName) => libraryLoader.Load(libraryName); static global::System.IntPtr GetFunctionByName(string functionName, global::System.IntPtr libraryHandle) - { - throw new NotImplementedException(); - } + => libraryLoader.GetFunction(libraryHandle, functionName); internal static delegate* unmanaged[Cdecl] PyDictProxy_New { get; } internal static delegate* unmanaged[Cdecl] Py_IncRef { get; } From 2498d474a3ae8e84d505680de5a467be5b5c4e9f Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 21:48:07 -0800 Subject: [PATCH 025/151] always explicitly specify the way strings are marshaled --- src/embed_tests/References.cs | 4 +- src/embed_tests/TestDomainReload.cs | 9 +- src/embed_tests/TestRuntime.cs | 8 +- src/embed_tests/TestTypeManager.cs | 65 ----- src/runtime/CustomMarshaler.cs | 11 +- src/runtime/Python.Runtime.csproj | 4 + src/runtime/exceptions.cs | 6 +- src/runtime/native/PyCompilerFlags.cs | 13 + src/runtime/native/StrPtr.cs | 73 +++++ src/runtime/pyint.cs | 2 +- src/runtime/pyobject.cs | 2 +- src/runtime/runtime.cs | 374 +++++++++++++++----------- src/runtime/runtime_data.cs | 11 +- 13 files changed, 336 insertions(+), 246 deletions(-) delete mode 100644 src/embed_tests/TestTypeManager.cs create mode 100644 src/runtime/native/PyCompilerFlags.cs create mode 100644 src/runtime/native/StrPtr.cs diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs index 1d29e85c7..417e743c0 100644 --- a/src/embed_tests/References.cs +++ b/src/embed_tests/References.cs @@ -23,7 +23,7 @@ public void Dispose() public void MoveToPyObject_SetsNull() { var dict = new PyDict(); - NewReference reference = Runtime.PyDict_Items(dict.Handle); + NewReference reference = Runtime.PyDict_Items(dict.Reference); try { Assert.IsFalse(reference.IsNull()); @@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull() public void CanBorrowFromNewReference() { var dict = new PyDict(); - NewReference reference = Runtime.PyDict_Items(dict.Handle); + NewReference reference = Runtime.PyDict_Items(dict.Reference); try { PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference)); diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index f8445edb4..e4479da18 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName) // assembly (and Python .NET) to reside var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Soft); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL); // From now on use the Proxy to call into the new assembly theProxy.RunPython(); @@ -400,7 +400,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Reload); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); arg = caller.Execute(arg); @@ -418,7 +418,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Reload); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); caller.Execute(arg); @@ -478,8 +478,9 @@ public static void RunPython() private static IntPtr _state; - public static void InitPython(ShutdownMode mode) + public static void InitPython(ShutdownMode mode, string dllName) { + PyRuntime.PythonDLL = dllName; PythonEngine.Initialize(mode: mode); _state = PythonEngine.BeginAllowThreads(); } diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index cde5dd6fa..59c66cc5e 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -96,13 +96,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. var threading = Runtime.Runtime.PyImport_ImportModule("threading"); Exceptions.ErrorCheck(threading); - var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading)); Exceptions.ErrorCheck(threadingDict); var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); - if (lockType == IntPtr.Zero) + if (lockType.IsNull) throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); - var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0)); + var args = Runtime.Runtime.PyTuple_New(0); + var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args); + Runtime.Runtime.XDecref(args); Exceptions.ErrorCheck(lockInstance); Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index 43155e1bf..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using Python.Runtime.Platform; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - Runtime.Runtime.Shutdown(); - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); - Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = NativeCodePageHelper.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - /* if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } */ - } - } -} diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 0cbbbaba2..4814e6c0b 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -41,8 +41,9 @@ public int GetNativeDataSize() /// internal class UcsMarshaler : MarshalerBase { + internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); - private static readonly Encoding PyEncoding = Runtime.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -91,13 +92,13 @@ public static int GetUnicodeByteLength(IntPtr p) var len = 0; while (true) { - int c = Runtime._UCS == 2 + int c = _UCS == 2 ? Marshal.ReadInt16(p, len * 2) : Marshal.ReadInt32(p, len * 4); if (c == 0) { - return len * Runtime._UCS; + return len * _UCS; } checked { @@ -147,7 +148,7 @@ public static string PtrToPy3UnicodePy2String(IntPtr p) internal class StrArrayMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrArrayMarshaler(); - private static readonly Encoding PyEncoding = Runtime.PyEncoding; + private static readonly Encoding PyEncoding = UcsMarshaler.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -159,7 +160,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) } int totalStrLength = argv.Sum(arg => arg.Length + 1); - int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS; + int memSize = argv.Length * IntPtr.Size + totalStrLength * UcsMarshaler._UCS; IntPtr mem = Marshal.AllocHGlobal(memSize); try diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index a09b0ff78..44ec759d7 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -31,5 +31,9 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 9091fd071..afd0bc14e 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -194,14 +194,16 @@ internal static void SetArgsAndCause(IntPtr ob) /// Shortcut for (pointer == NULL) -> throw PythonException /// /// Pointer to a Python object - internal static void ErrorCheck(IntPtr pointer) + internal static void ErrorCheck(BorrowedReference pointer) { - if (pointer == IntPtr.Zero) + if (pointer.IsNull) { throw new PythonException(); } } + internal static void ErrorCheck(IntPtr pointer) => ErrorCheck(new BorrowedReference(pointer)); + /// /// Shortcut for (pointer == NULL or ErrorOccurred()) -> throw PythonException /// diff --git a/src/runtime/native/PyCompilerFlags.cs b/src/runtime/native/PyCompilerFlags.cs new file mode 100644 index 000000000..9e8242c58 --- /dev/null +++ b/src/runtime/native/PyCompilerFlags.cs @@ -0,0 +1,13 @@ +using System; + +namespace Python.Runtime.Native +{ + [Flags] + enum PyCompilerFlags + { + SOURCE_IS_UTF8 = 0x0100, + DONT_IMPLY_DEDENT = 0x0200, + ONLY_AST = 0x0400, + IGNORE_COOKIE = 0x0800, + } +} diff --git a/src/runtime/native/StrPtr.cs b/src/runtime/native/StrPtr.cs new file mode 100644 index 000000000..99aa35ddd --- /dev/null +++ b/src/runtime/native/StrPtr.cs @@ -0,0 +1,73 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime.Native +{ + [StructLayout(LayoutKind.Sequential)] + struct StrPtr : IDisposable + { + public IntPtr RawPointer { get; set; } + unsafe byte* Bytes => (byte*)this.RawPointer; + + public unsafe StrPtr(string value, Encoding encoding) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + if (encoding is null) throw new ArgumentNullException(nameof(encoding)); + + var bytes = encoding.GetBytes(value); + this.RawPointer = Marshal.AllocHGlobal(checked(bytes.Length + 1)); + try + { + Marshal.Copy(bytes, 0, this.RawPointer, bytes.Length); + this.Bytes[bytes.Length] = 0; + } + catch + { + this.Dispose(); + throw; + } + } + + public unsafe string? ToString(Encoding encoding) + { + if (encoding is null) throw new ArgumentNullException(nameof(encoding)); + if (this.RawPointer == IntPtr.Zero) return null; + + return encoding.GetString((byte*)this.RawPointer, byteCount: checked((int)this.ByteCount)); + } + + public unsafe nuint ByteCount + { + get + { + if (this.RawPointer == IntPtr.Zero) throw new NullReferenceException(); + + nuint zeroIndex = 0; + while (this.Bytes[zeroIndex] != 0) + { + zeroIndex++; + } + return zeroIndex; + } + } + + public void Dispose() + { + if (this.RawPointer == IntPtr.Zero) + return; + + Marshal.FreeHGlobal(this.RawPointer); + this.RawPointer = IntPtr.Zero; + } + + internal static Encoding GetEncodingByPythonName(string pyEncodingName) + { + // https://stackoverflow.com/a/7798749/231238 + if (pyEncodingName == "mbcs") return Encoding.Default; + + return Encoding.GetEncoding(pyEncodingName); + } + } +} diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f8718cb95..7b02c68e5 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -149,7 +149,7 @@ public PyInt(sbyte value) : this((int)value) private static IntPtr FromString(string value) { - IntPtr val = Runtime.PyInt_FromString(value, IntPtr.Zero, 0); + IntPtr val = Runtime.PyLong_FromString(value, IntPtr.Zero, 0); PythonException.ThrowIfIsNull(val); return val; } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index d68a9905b..b9125f40d 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -257,7 +257,7 @@ public bool HasAttr(string name) { if (name == null) throw new ArgumentNullException(nameof(name)); - return Runtime.PyObject_HasAttrString(obj, name) != 0; + return Runtime.PyObject_HasAttrString(Reference, name) != 0; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index f51a18e46..616b2dfe3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -19,9 +19,6 @@ namespace Python.Runtime /// public unsafe class Runtime { - public static int UCS => _UCS; - internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; - public static string PythonDLL { get => _PythonDll; @@ -51,11 +48,6 @@ public static string PythonDLL public static int MainManagedThreadId { get; private set; } - /// - /// Encoding to use to convert Unicode to/from Managed to Native - /// - internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - public static ShutdownMode ShutdownMode { get; internal set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); @@ -212,7 +204,7 @@ private static void InitPyMembers() () => PyUnicodeType = IntPtr.Zero); XDecref(op); - op = PyBytes_FromString("bytes"); + op = EmptyPyBytes(); SetPyMember(ref PyBytesType, PyObject_Type(op), () => PyBytesType = IntPtr.Zero); XDecref(op); @@ -830,13 +822,20 @@ internal static unsafe long Refcount(IntPtr op) internal static IntPtr PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - public static int Py_Main( - int argc, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv - ) => Delegates.Py_Main(argc, argv -); + public static int Py_Main(int argc, string[] argv) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + return Delegates.Py_Main(argc, argvPtr); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } - internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); @@ -902,13 +901,20 @@ public static int Py_Main( internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); - - internal static int PyRun_SimpleString(string code) => Delegates.PyRun_SimpleString(code); + const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; - - internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) => Delegates.PyRun_String(code, st, globals, locals); + internal static int PyRun_SimpleString(string code) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); + } + + internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); + } - internal static IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals) => Delegates.PyEval_EvalCode(co, globals, locals); /// @@ -917,30 +923,17 @@ public static int Py_Main( /// internal static IntPtr Py_CompileString(string str, string file, int start) { - return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + using var strPtr = new StrPtr(str, Encoding.UTF8); + using var fileObj = new PyString(file); + return Delegates.Py_CompileStringObject(strPtr, fileObj.Reference, start, Utf8String, -1); } - /// - /// Return value: New reference. - /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. - /// - internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + internal static IntPtr PyImport_ExecCodeModule(string name, IntPtr code) { - return Py_CompileStringExFlags(str, file, start, flags, -1); + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ExecCodeModule(namePtr, code); } - /// - /// Return value: New reference. - /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). - /// - /// - - internal static IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize) => Delegates.Py_CompileStringExFlags(str, file, start, flags, optimize); - - - internal static IntPtr PyImport_ExecCodeModule(string name, IntPtr code) => Delegates.PyImport_ExecCodeModule(name, code); - - internal static IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod) => Delegates.PyCFunction_NewEx(ml, self, mod); @@ -1009,7 +1002,11 @@ internal static bool PyObject_IsIterable(IntPtr pointer) } - internal static int PyObject_HasAttrString(IntPtr pointer, string name) => Delegates.PyObject_HasAttrString(pointer, name); + internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_HasAttrString(pointer, namePtr); + } internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) { @@ -1027,10 +1024,13 @@ internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) internal static IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name) => Delegates.PyObject_GetAttrString(pointer, name); - - internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) => Delegates.PyObject_SetAttrString(pointer, name, value); - + internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(pointer, namePtr, value); + } + internal static int PyObject_HasAttr(IntPtr pointer, IntPtr name) => Delegates.PyObject_HasAttr(pointer, name); @@ -1145,9 +1145,12 @@ internal static long PyObject_Size(IntPtr pointer) internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); - internal static IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format) => Delegates.PyBuffer_SizeFromFormat(format); + internal static IntPtr PyBuffer_SizeFromFormat(string format) + { + using var formatPtr = new StrPtr(format, Encoding.ASCII); + return Delegates.PyBuffer_SizeFromFormat(formatPtr); + } - internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); @@ -1212,8 +1215,6 @@ internal static IntPtr PyInt_FromInt64(long value) internal static int PyInt_AsLong(IntPtr value) => Delegates.PyInt_AsLong(value); - internal static IntPtr PyInt_FromString(string value, IntPtr end, int radix) => Delegates.PyInt_FromString(value, end, radix); - internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; @@ -1246,7 +1247,11 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) internal static IntPtr PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); - internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) => Delegates.PyLong_FromString(value, end, radix); + internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) + { + using var valPtr = new StrPtr(value, Encoding.UTF8); + return Delegates.PyLong_FromString(valPtr, end, radix); + } @@ -1478,11 +1483,17 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { - return PyUnicode_FromKindAndData(_UCS, value, value.Length); + fixed(char* ptr = value) + return PyUnicode_FromKindAndData(2, (IntPtr)ptr, value.Length); } - internal static IntPtr PyBytes_FromString(string op) => Delegates.PyBytes_FromString(op); + internal static IntPtr EmptyPyBytes() + { + byte* bytes = stackalloc byte[1]; + bytes[0] = 0; + return Delegates.PyBytes_FromString((IntPtr)bytes); + } internal static long PyBytes_Size(IntPtr op) { @@ -1520,22 +1531,19 @@ internal static bool PyUnicode_Check(IntPtr ob) internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) + internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size) { return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); } - private static IntPtr PyUnicode_FromKindAndData( - int kind, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size - ) => Delegates.PyUnicode_FromKindAndData(kind, s, size -); + private static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, IntPtr size) + => Delegates.PyUnicode_FromKindAndData(kind, s, size); internal static IntPtr PyUnicode_FromUnicode(string s, long size) { - return PyUnicode_FromKindAndData(_UCS, s, size); + fixed(char* ptr = s) + return PyUnicode_FromKindAndData(2, (IntPtr)ptr, size); } @@ -1551,8 +1559,10 @@ internal static long PyUnicode_GetSize(IntPtr ob) internal static IntPtr PyUnicode_AsUnicode(IntPtr ob) => Delegates.PyUnicode_AsUnicode(ob); + internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); + + - internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); internal static IntPtr PyUnicode_FromString(string s) @@ -1561,9 +1571,12 @@ internal static IntPtr PyUnicode_FromString(string s) } - internal static IntPtr PyUnicode_InternFromString(string s) => Delegates.PyUnicode_InternFromString(s); + internal static IntPtr PyUnicode_InternFromString(string s) + { + using var ptr = new StrPtr(s, Encoding.UTF8); + return Delegates.PyUnicode_InternFromString(ptr); + } - internal static int PyUnicode_Compare(IntPtr left, IntPtr right) => Delegates.PyUnicode_Compare(left, right); internal static string GetManagedString(in BorrowedReference borrowedReference) @@ -1587,13 +1600,12 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { - IntPtr p = PyUnicode_AsUnicode(op); + using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); int length = (int)PyUnicode_GetSize(op); - - int size = length * _UCS; - var buffer = new byte[size]; - Marshal.Copy(p, buffer, 0, size); - return PyEncoding.GetString(buffer, 0, size); + char* codePoints = (char*)PyBytes_AS_STRING(p.DangerousGetAddress()); + return new string(codePoints, + startIndex: 1, // skip BOM + length: length); } return null; @@ -1630,8 +1642,12 @@ internal static IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key) /// internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); - internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) => Delegates.PyDict_GetItemString(pointer, key); - + internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) + { + using var keyStr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_GetItemString(pointer, keyStr); + } + internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); /// @@ -1651,19 +1667,27 @@ internal static IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key) /// /// Return 0 on success or -1 on failure. /// - internal static int PyDict_SetItemString(IntPtr dict, string key, IntPtr value) => Delegates.PyDict_SetItemString(new BorrowedReference(dict), key, new BorrowedReference(value)); + internal static int PyDict_SetItemString(IntPtr dict, string key, IntPtr value) + => PyDict_SetItemString(new BorrowedReference(dict), key, new BorrowedReference(value)); + /// /// Return 0 on success or -1 on failure. /// - internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) => Delegates.PyDict_SetItemString(dict, key, value); - + internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_SetItemString(dict, keyPtr, value); + } internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); - internal static int PyDict_DelItemString(BorrowedReference pointer, string key) => Delegates.PyDict_DelItemString(pointer, key); + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_DelItemString(pointer, keyPtr); + } - internal static int PyMapping_HasKey(IntPtr pointer, IntPtr key) => Delegates.PyMapping_HasKey(pointer, key); @@ -1854,17 +1878,21 @@ internal static IntPtr PyIter_Next(IntPtr pointer) // Python module API //==================================================================== - - internal static NewReference PyModule_New(string name) => Delegates.PyModule_New(name); - - internal static string PyModule_GetName(IntPtr module) => Delegates.PyModule_GetName(module); + internal static NewReference PyModule_New(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_New(namePtr); + } + + internal static string PyModule_GetName(IntPtr module) + => Delegates.PyModule_GetName(module).ToString(Encoding.UTF8); - internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); - internal static string PyModule_GetFilename(IntPtr module) => Delegates.PyModule_GetFilename(module); + internal static string PyModule_GetFilename(IntPtr module) + => Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8); #if PYTHON_WITH_PYDEBUG [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] @@ -1879,37 +1907,55 @@ internal static IntPtr PyIter_Next(IntPtr pointer) /// /// Return value: New reference. /// - - internal static IntPtr PyImport_ImportModule(string name) => Delegates.PyImport_ImportModule(name); - + internal static IntPtr PyImport_ImportModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ImportModule(namePtr); + } + internal static IntPtr PyImport_ReloadModule(IntPtr module) => Delegates.PyImport_ReloadModule(module); - internal static BorrowedReference PyImport_AddModule(string name) => Delegates.PyImport_AddModule(name); + internal static BorrowedReference PyImport_AddModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_AddModule(namePtr); + } - internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); - internal static void PySys_SetArgvEx( - int argc, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, - int updatepath - ) => Delegates.PySys_SetArgvEx(argc, argv, updatepath -); + internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + Delegates.PySys_SetArgvEx(argc, argvPtr, updatepath); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } /// /// Return value: Borrowed reference. /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. /// - - internal static BorrowedReference PySys_GetObject(string name) => Delegates.PySys_GetObject(name); + internal static BorrowedReference PySys_GetObject(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_GetObject(namePtr); + } - [Obsolete] - internal static int PySys_SetObject(string name, IntPtr ob) => Delegates.PySys_SetObject(name, new BorrowedReference(ob)); - internal static int PySys_SetObject(string name, BorrowedReference ob) => Delegates.PySys_SetObject(name, ob); + internal static int PySys_SetObject(string name, BorrowedReference ob) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_SetObject(namePtr, ob); + } //==================================================================== @@ -2013,9 +2059,12 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) //==================================================================== - internal static void PyErr_SetString(IntPtr ob, string message) => Delegates.PyErr_SetString(ob, message); + internal static void PyErr_SetString(IntPtr ob, string message) + { + using var msgPtr = new StrPtr(message, Encoding.UTF8); + Delegates.PyErr_SetString(ob, msgPtr); + } - internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); @@ -2121,13 +2170,15 @@ internal static void Py_CLEAR(ref IntPtr ob) // Python Capsules API //==================================================================== - - internal static NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor) => Delegates.PyCapsule_New(pointer, name, destructor); - - internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name) => Delegates.PyCapsule_GetPointer(capsule, name); + internal static NewReference PyCapsule_New(IntPtr pointer, IntPtr name, IntPtr destructor) + => Delegates.PyCapsule_New(pointer, name, destructor); + + internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr name) + { + return Delegates.PyCapsule_GetPointer(capsule, name); + } - internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); //==================================================================== @@ -2187,7 +2238,7 @@ internal static IntPtr GetBuiltins() private static class Delegates { - static readonly ILibraryLoader libraryLoader = LibraryLoader.Get(); + static readonly ILibraryLoader libraryLoader = LibraryLoader.Instance; static Delegates() { @@ -2210,7 +2261,7 @@ static Delegates() PyGILState_Ensure = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Ensure), GetUnmanagedDll(_PythonDll)); PyGILState_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Release), GetUnmanagedDll(_PythonDll)); PyGILState_GetThisThreadState = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_GetThisThreadState), GetUnmanagedDll(_PythonDll)); - Py_Main = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Main), GetUnmanagedDll(_PythonDll)); + Py_Main = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Main), GetUnmanagedDll(_PythonDll)); PyEval_InitThreads = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_InitThreads), GetUnmanagedDll(_PythonDll)); PyEval_ThreadsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ThreadsInitialized), GetUnmanagedDll(_PythonDll)); PyEval_AcquireLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireLock), GetUnmanagedDll(_PythonDll)); @@ -2233,17 +2284,17 @@ static Delegates() Py_GetCopyright = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCopyright), GetUnmanagedDll(_PythonDll)); Py_GetCompiler = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCompiler), GetUnmanagedDll(_PythonDll)); Py_GetBuildInfo = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetBuildInfo), GetUnmanagedDll(_PythonDll)); - PyRun_SimpleString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_SimpleString), GetUnmanagedDll(_PythonDll)); - PyRun_String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_String), GetUnmanagedDll(_PythonDll)); + PyRun_SimpleStringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_SimpleStringFlags), GetUnmanagedDll(_PythonDll)); + PyRun_StringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_StringFlags), GetUnmanagedDll(_PythonDll)); PyEval_EvalCode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_EvalCode), GetUnmanagedDll(_PythonDll)); - Py_CompileStringExFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringExFlags), GetUnmanagedDll(_PythonDll)); - PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); + Py_CompileStringObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringObject), GetUnmanagedDll(_PythonDll)); + PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); PyCFunction_NewEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_NewEx), GetUnmanagedDll(_PythonDll)); PyCFunction_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_Call), GetUnmanagedDll(_PythonDll)); PyMethod_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_New), GetUnmanagedDll(_PythonDll)); - PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); - PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); PyObject_GetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttr), GetUnmanagedDll(_PythonDll)); PyObject_SetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttr), GetUnmanagedDll(_PythonDll)); @@ -2267,7 +2318,14 @@ static Delegates() PyObject_Dir = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll)); PyObject_GetBuffer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll)); PyBuffer_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll)); - PyBuffer_SizeFromFormat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_SizeFromFormat), GetUnmanagedDll(_PythonDll)); + try + { + PyBuffer_SizeFromFormat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_SizeFromFormat), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + // only in 3.9+ + } PyBuffer_IsContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_IsContiguous), GetUnmanagedDll(_PythonDll)); PyBuffer_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll)); PyBuffer_FromContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FromContiguous), GetUnmanagedDll(_PythonDll)); @@ -2280,14 +2338,13 @@ static Delegates() PyNumber_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Check), GetUnmanagedDll(_PythonDll)); PyInt_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromLong", GetUnmanagedDll(_PythonDll)); PyInt_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLong", GetUnmanagedDll(_PythonDll)); - PyInt_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromString", GetUnmanagedDll(_PythonDll)); PyLong_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLong), GetUnmanagedDll(_PythonDll)); PyLong_FromUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); PyLong_FromUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); PyLong_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromDouble), GetUnmanagedDll(_PythonDll)); PyLong_FromLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLongLong), GetUnmanagedDll(_PythonDll)); PyLong_FromUnsignedLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromUnsignedLongLong), GetUnmanagedDll(_PythonDll)); - PyLong_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromString), GetUnmanagedDll(_PythonDll)); + PyLong_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromString), GetUnmanagedDll(_PythonDll)); PyLong_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsLong), GetUnmanagedDll(_PythonDll)); PyLong_AsUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); PyLong_AsUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); @@ -2338,27 +2395,28 @@ static Delegates() _PySequence_Count = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Count", GetUnmanagedDll(_PythonDll)); PySequence_Tuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Tuple), GetUnmanagedDll(_PythonDll)); PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); - PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); + PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); - PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll)); PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUTF16String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF16String), GetUnmanagedDll(_PythonDll)); PyUnicode_FromOrdinal = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromOrdinal), GetUnmanagedDll(_PythonDll)); - PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); + PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); PyUnicode_Compare = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_Compare), GetUnmanagedDll(_PythonDll)); PyDict_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_New), GetUnmanagedDll(_PythonDll)); PyDict_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Next), GetUnmanagedDll(_PythonDll)); PyDict_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItem), GetUnmanagedDll(_PythonDll)); - PyDict_GetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_GetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemString), GetUnmanagedDll(_PythonDll)); PyDict_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItem), GetUnmanagedDll(_PythonDll)); - PyDict_SetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_SetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItemString), GetUnmanagedDll(_PythonDll)); PyDict_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItem), GetUnmanagedDll(_PythonDll)); - PyDict_DelItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItemString), GetUnmanagedDll(_PythonDll)); + PyDict_DelItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItemString), GetUnmanagedDll(_PythonDll)); PyMapping_HasKey = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMapping_HasKey), GetUnmanagedDll(_PythonDll)); PyDict_Keys = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Keys), GetUnmanagedDll(_PythonDll)); PyDict_Values = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Values), GetUnmanagedDll(_PythonDll)); @@ -2387,19 +2445,19 @@ static Delegates() PyTuple_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_GetSlice), GetUnmanagedDll(_PythonDll)); PyTuple_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_Size), GetUnmanagedDll(_PythonDll)); PyIter_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyIter_Next), GetUnmanagedDll(_PythonDll)); - PyModule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_New), GetUnmanagedDll(_PythonDll)); - PyModule_GetName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll)); + PyModule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_New), GetUnmanagedDll(_PythonDll)); + PyModule_GetName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll)); PyModule_GetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll)); - PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); + PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); - PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); + PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); - PyImport_AddModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_AddModule), GetUnmanagedDll(_PythonDll)); + PyImport_AddModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_AddModule), GetUnmanagedDll(_PythonDll)); PyImport_GetModuleDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_GetModuleDict), GetUnmanagedDll(_PythonDll)); - PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); - PySys_GetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_GetObject), GetUnmanagedDll(_PythonDll)); - PySys_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetObject), GetUnmanagedDll(_PythonDll)); + PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); + PySys_GetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_GetObject), GetUnmanagedDll(_PythonDll)); + PySys_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetObject), GetUnmanagedDll(_PythonDll)); PyType_Modified = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Modified), GetUnmanagedDll(_PythonDll)); PyType_IsSubtype = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_IsSubtype), GetUnmanagedDll(_PythonDll)); PyType_GenericNew = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericNew), GetUnmanagedDll(_PythonDll)); @@ -2416,7 +2474,7 @@ static Delegates() PyMem_Malloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Malloc), GetUnmanagedDll(_PythonDll)); PyMem_Realloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Realloc), GetUnmanagedDll(_PythonDll)); PyMem_Free = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Free), GetUnmanagedDll(_PythonDll)); - PyErr_SetString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetString), GetUnmanagedDll(_PythonDll)); + PyErr_SetString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetString), GetUnmanagedDll(_PythonDll)); PyErr_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetObject), GetUnmanagedDll(_PythonDll)); PyErr_SetFromErrno = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetFromErrno), GetUnmanagedDll(_PythonDll)); PyErr_SetNone = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetNone), GetUnmanagedDll(_PythonDll)); @@ -2431,8 +2489,8 @@ static Delegates() PyCell_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Get), GetUnmanagedDll(_PythonDll)); PyCell_Set = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Set), GetUnmanagedDll(_PythonDll)); PyGC_Collect = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGC_Collect), GetUnmanagedDll(_PythonDll)); - PyCapsule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_New), GetUnmanagedDll(_PythonDll)); - PyCapsule_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_GetPointer), GetUnmanagedDll(_PythonDll)); + PyCapsule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_New), GetUnmanagedDll(_PythonDll)); + PyCapsule_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_GetPointer), GetUnmanagedDll(_PythonDll)); PyCapsule_SetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_SetPointer), GetUnmanagedDll(_PythonDll)); PyMethod_Self = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Self), GetUnmanagedDll(_PythonDll)); PyMethod_Function = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Function), GetUnmanagedDll(_PythonDll)); @@ -2471,7 +2529,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyGILState_Ensure { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Release { get; } internal static delegate* unmanaged[Cdecl] PyGILState_GetThisThreadState { get; } - internal static delegate* unmanaged[Cdecl] Py_Main { get; } + internal static delegate* unmanaged[Cdecl] Py_Main { get; } internal static delegate* unmanaged[Cdecl] PyEval_InitThreads { get; } internal static delegate* unmanaged[Cdecl] PyEval_ThreadsInitialized { get; } internal static delegate* unmanaged[Cdecl] PyEval_AcquireLock { get; } @@ -2494,17 +2552,17 @@ static Delegates() internal static delegate* unmanaged[Cdecl] Py_GetCopyright { get; } internal static delegate* unmanaged[Cdecl] Py_GetCompiler { get; } internal static delegate* unmanaged[Cdecl] Py_GetBuildInfo { get; } - internal static delegate* unmanaged[Cdecl] PyRun_SimpleString { get; } - internal static delegate* unmanaged[Cdecl] PyRun_String { get; } + internal static delegate* unmanaged[Cdecl] PyRun_SimpleStringFlags { get; } + internal static delegate* unmanaged[Cdecl] PyRun_StringFlags { get; } internal static delegate* unmanaged[Cdecl] PyEval_EvalCode { get; } - internal static delegate* unmanaged[Cdecl] Py_CompileStringExFlags { get; } - internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } + internal static delegate* unmanaged[Cdecl] Py_CompileStringObject { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } internal static delegate* unmanaged[Cdecl] PyCFunction_NewEx { get; } internal static delegate* unmanaged[Cdecl] PyCFunction_Call { get; } internal static delegate* unmanaged[Cdecl] PyMethod_New { get; } - internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } - internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_SetAttr { get; } @@ -2528,7 +2586,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_Dir { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetBuffer { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } - internal static delegate* unmanaged[Cdecl] PyBuffer_SizeFromFormat { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_SizeFromFormat { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_IsContiguous { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_GetPointer { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_FromContiguous { get; } @@ -2541,14 +2599,13 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyNumber_Check { get; } internal static delegate* unmanaged[Cdecl] PyInt_FromLong { get; } internal static delegate* unmanaged[Cdecl] PyInt_AsLong { get; } - internal static delegate* unmanaged[Cdecl] PyInt_FromString { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromLong { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong32 { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong64 { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromDouble { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromLongLong { get; } internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLongLong { get; } - internal static delegate* unmanaged[Cdecl] PyLong_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromString { get; } internal static delegate* unmanaged[Cdecl] PyLong_AsLong { get; } internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong32 { get; } internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong64 { get; } @@ -2599,27 +2656,28 @@ static Delegates() internal static delegate* unmanaged[Cdecl] _PySequence_Count { get; } internal static delegate* unmanaged[Cdecl] PySequence_Tuple { get; } internal static delegate* unmanaged[Cdecl] PySequence_List { get; } - internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromStringAndSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_FromKindAndData { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromKindAndData { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF16String { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromOrdinal { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_Compare { get; } internal static delegate* unmanaged[Cdecl] PyDict_New { get; } internal static delegate* unmanaged[Cdecl] PyDict_Next { get; } internal static delegate* unmanaged[Cdecl] PyDict_GetItem { get; } - internal static delegate* unmanaged[Cdecl] PyDict_GetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItemString { get; } internal static delegate* unmanaged[Cdecl] PyDict_SetItem { get; } - internal static delegate* unmanaged[Cdecl] PyDict_SetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_SetItemString { get; } internal static delegate* unmanaged[Cdecl] PyDict_DelItem { get; } - internal static delegate* unmanaged[Cdecl] PyDict_DelItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_DelItemString { get; } internal static delegate* unmanaged[Cdecl] PyMapping_HasKey { get; } internal static delegate* unmanaged[Cdecl] PyDict_Keys { get; } internal static delegate* unmanaged[Cdecl] PyDict_Values { get; } @@ -2648,19 +2706,19 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyTuple_GetSlice { get; } internal static delegate* unmanaged[Cdecl] PyTuple_Size { get; } internal static delegate* unmanaged[Cdecl] PyIter_Next { get; } - internal static delegate* unmanaged[Cdecl] PyModule_New { get; } - internal static delegate* unmanaged[Cdecl] PyModule_GetName { get; } + internal static delegate* unmanaged[Cdecl] PyModule_New { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetName { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } - internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } - internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } - internal static delegate* unmanaged[Cdecl] PyImport_AddModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_AddModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_GetModuleDict { get; } - internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } - internal static delegate* unmanaged[Cdecl] PySys_GetObject { get; } - internal static delegate* unmanaged[Cdecl] PySys_SetObject { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } + internal static delegate* unmanaged[Cdecl] PySys_GetObject { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetObject { get; } internal static delegate* unmanaged[Cdecl] PyType_Modified { get; } internal static delegate* unmanaged[Cdecl] PyType_IsSubtype { get; } internal static delegate* unmanaged[Cdecl] PyType_GenericNew { get; } @@ -2677,7 +2735,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyMem_Malloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Realloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Free { get; } - internal static delegate* unmanaged[Cdecl] PyErr_SetString { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetString { get; } internal static delegate* unmanaged[Cdecl] PyErr_SetObject { get; } internal static delegate* unmanaged[Cdecl] PyErr_SetFromErrno { get; } internal static delegate* unmanaged[Cdecl] PyErr_SetNone { get; } @@ -2692,8 +2750,8 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyCell_Get { get; } internal static delegate* unmanaged[Cdecl] PyCell_Set { get; } internal static delegate* unmanaged[Cdecl] PyGC_Collect { get; } - internal static delegate* unmanaged[Cdecl] PyCapsule_New { get; } - internal static delegate* unmanaged[Cdecl] PyCapsule_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_New { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_GetPointer { get; } internal static delegate* unmanaged[Cdecl] PyCapsule_SetPointer { get; } internal static delegate* unmanaged[Cdecl] PyMethod_Self { get; } internal static delegate* unmanaged[Cdecl] PyMethod_Function { get; } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 1d1cd3ad0..abf057cef 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -39,7 +39,7 @@ static void ClearCLRData () BorrowedReference capsule = PySys_GetObject("clr_data"); if (!capsule.IsNull) { - IntPtr oldData = PyCapsule_GetPointer(capsule, null); + IntPtr oldData = PyCapsule_GetPointer(capsule, IntPtr.Zero); PyMem_Free(oldData); PyCapsule_SetPointer(capsule, IntPtr.Zero); } @@ -85,8 +85,9 @@ internal static void Stash() Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); ClearCLRData(); - NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); - PySys_SetObject("clr_data", capsule.DangerousGetAddress()); + #warning this leaks memory in mem + NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); + PySys_SetObject("clr_data", capsule); // Let the dictionary own the reference capsule.Dispose(); } @@ -110,7 +111,7 @@ private static void RestoreRuntimeDataImpl() { return; } - IntPtr mem = PyCapsule_GetPointer(capsule, null); + IntPtr mem = PyCapsule_GetPointer(capsule, IntPtr.Zero); int length = (int)Marshal.ReadIntPtr(mem); byte[] data = new byte[length]; Marshal.Copy(mem + IntPtr.Size, data, 0, length); @@ -143,7 +144,7 @@ public static bool HasStashData() public static void ClearStash() { - PySys_SetObject("clr_data", IntPtr.Zero); + PySys_SetObject("clr_data", default); } static bool CheckSerializable (object o) From 70fc803e824d6d842e10a75667f15ce393085d8a Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 22:05:18 -0800 Subject: [PATCH 026/151] CI: figure out DLL name from environment --- .github/workflows/main.yml | 1 + src/runtime/runtime.cs | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 688d65c04..a5c1c40f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,7 @@ jobs: env: PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} + PYTHONNET_PYVER: ${{ matrix.PYTHON }} steps: - name: Set Environment on macOS diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 616b2dfe3..90b280a34 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -9,6 +9,7 @@ using Python.Runtime.Native; using Python.Runtime.Platform; using System.Linq; +using static System.FormattableString; namespace Python.Runtime { @@ -30,7 +31,29 @@ public static string PythonDLL } } - static string _PythonDll; + static string _PythonDll = GetDefaultDllName(); + private static string GetDefaultDllName() + { + string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); + if (dll is not null) return dll; + + string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); + if (!Version.TryParse(verString, out var version)) return null; + + return GetDefaultDllName(version); + } + + private static string GetDefaultDllName(Version version) + { + string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Invariant($"{version.Major}{version.Minor}") + : Invariant($"{version.Major}.{version.Minor}"); + string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" + : ".so"; + return prefix + "python" + suffix + ext; + } // set to true when python is finalizing internal static object IsFinalizingLock = new object(); From 28a5dab693c12f00674d9456df24b61d6e46bb7b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 22:59:17 -0800 Subject: [PATCH 027/151] use Roslyn preview in CI workaround for https://github.com/dotnet/roslyn/issues/49760 --- Directory.Build.props | 6 +++++- src/runtime/Python.Runtime.csproj | 4 ---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 5ad0c0e77..4406a1870 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,7 +8,11 @@ - + + all + runtime; build; native; contentfiles; analyzers + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 44ec759d7..a09b0ff78 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -31,9 +31,5 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - From c75229a1bee215aae22d62723e03cc7919734252 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 23:18:38 -0800 Subject: [PATCH 028/151] fixed Linux and Mac DLL loaders breaking dll path --- src/runtime/platform/LibraryLoader.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index 193dff274..d3ddb652d 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -49,13 +49,12 @@ class LinuxLoader : ILibraryLoader public IntPtr Load(string dllToLoad) { - var filename = $"lib{dllToLoad}.so"; ClearError(); - var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + var res = dlopen(dllToLoad, RTLD_NOW | RTLD_GLOBAL); if (res == IntPtr.Zero) { var err = GetError(); - throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + throw new DllNotFoundException($"Could not load {dllToLoad} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); } return res; @@ -120,13 +119,12 @@ class DarwinLoader : ILibraryLoader public IntPtr Load(string dllToLoad) { - var filename = $"lib{dllToLoad}.dylib"; ClearError(); - var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + var res = dlopen(dllToLoad, RTLD_NOW | RTLD_GLOBAL); if (res == IntPtr.Zero) { var err = GetError(); - throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + throw new DllNotFoundException($"Could not load {dllToLoad} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); } return res; From a0a1dc1013adabfdfffc88c8816cf649f75d037d Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 23:26:28 -0800 Subject: [PATCH 029/151] correctly detect DLL on *nix when running from Python --- src/runtime/runtime.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 90b280a34..13a137755 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -37,6 +37,12 @@ private static string GetDefaultDllName() string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); if (dll is not null) return dll; + try + { + LibraryLoader.Instance.GetFunction(IntPtr.Zero, "PyUnicode_GetMax"); + return null; + } catch (MissingMethodException) { } + string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); if (!Version.TryParse(verString, out var version)) return null; From 1b887838fc2f24331014bf666fdca0fb1b2cf9ea Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 23:42:25 -0800 Subject: [PATCH 030/151] Windows library loader: add support for hModule == 0 --- src/runtime/platform/LibraryLoader.cs | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index d3ddb652d..099b9d6cd 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -1,5 +1,7 @@ using System; using System.ComponentModel; +using System.Diagnostics; +using System.Linq; using System.Runtime.InteropServices; namespace Python.Runtime.Platform @@ -195,6 +197,15 @@ public IntPtr Load(string dllToLoad) public IntPtr GetFunction(IntPtr hModule, string procedureName) { + if (hModule == IntPtr.Zero) + { + foreach(var module in GetAllModules()) + { + var func = GetProcAddress(module, procedureName); + if (func != IntPtr.Zero) return func; + } + } + var res = WindowsLoader.GetProcAddress(hModule, procedureName); if (res == IntPtr.Zero) throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); @@ -203,6 +214,24 @@ public IntPtr GetFunction(IntPtr hModule, string procedureName) public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + static IntPtr[] GetAllModules() + { + var self = Process.GetCurrentProcess().Handle; + + uint bytes = 0; + var result = new IntPtr[0]; + if (!EnumProcessModules(self, result, bytes, out var needsBytes)) + throw new Win32Exception(); + while (bytes < needsBytes) + { + bytes = needsBytes; + result = new IntPtr[bytes / IntPtr.Size]; + if (!EnumProcessModules(self, result, bytes, out needsBytes)) + throw new Win32Exception(); + } + return result.Take((int)(needsBytes / IntPtr.Size)).ToArray(); + } + [DllImport(NativeDll, SetLastError = true)] static extern IntPtr LoadLibrary(string dllToLoad); @@ -211,5 +240,8 @@ public IntPtr GetFunction(IntPtr hModule, string procedureName) [DllImport(NativeDll)] static extern bool FreeLibrary(IntPtr hModule); + + [DllImport("Psapi.dll", SetLastError = true)] + static extern bool EnumProcessModules(IntPtr hProcess, [In, Out] IntPtr[] lphModule, uint lphModuleByteCount, out uint byteCountNeeded); } } From 2c1aaefc09c4ab9e64329029ec45c8f3c7fb3e02 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 21 Jan 2021 23:57:47 -0800 Subject: [PATCH 031/151] fix dll loading in tests --- src/runtime/runtime.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 13a137755..92bc53a29 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -54,7 +54,7 @@ private static string GetDefaultDllName(Version version) string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? Invariant($"{version.Major}{version.Minor}") - : Invariant($"{version.Major}.{version.Minor}"); + : Invariant($"{version.Major}.{version.Minor}m"); string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" : ".so"; @@ -2534,7 +2534,11 @@ static Delegates() PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); } - static global::System.IntPtr GetUnmanagedDll(string libraryName) => libraryLoader.Load(libraryName); + static global::System.IntPtr GetUnmanagedDll(string libraryName) + { + if (libraryName is null) return IntPtr.Zero; + return libraryLoader.Load(libraryName); + } static global::System.IntPtr GetFunctionByName(string functionName, global::System.IntPtr libraryHandle) => libraryLoader.GetFunction(libraryHandle, functionName); From 39e41d07bad264b365365e58dedcb7e0eefa7dde Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 00:08:23 -0800 Subject: [PATCH 032/151] mentiond PythonDLL in changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 110d7c1c3..d64f3cdef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ details about the cause of the failure when .NET expects an integer [#1342][i1342] - More specific error messages for method argument mismatch - BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. +- BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name +or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. ### Fixed From 17040fe130450a95edb863104e5f91a43497775e Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 10:36:03 -0800 Subject: [PATCH 033/151] set PYDLL in AppVeyor --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 1ad673ede..21e816f38 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,6 +30,7 @@ environment: init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% + - set PYTHONNET_PYDLL=python%PY_VER%.dll - set PYTHON=C:\PYTHON%PY_VER% - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64) From b7410b63e24d5714deaff0355e84d349ee486c5f Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 10:44:58 -0800 Subject: [PATCH 034/151] revert automatically added 'm' suffix for *nix default dll name --- src/runtime/runtime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 92bc53a29..f5d5cb371 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -54,7 +54,7 @@ private static string GetDefaultDllName(Version version) string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? Invariant($"{version.Major}{version.Minor}") - : Invariant($"{version.Major}.{version.Minor}m"); + : Invariant($"{version.Major}.{version.Minor}"); string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" : ".so"; From 275cae92abeb1ea3ce5605518a59624e81165710 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 10:45:51 -0800 Subject: [PATCH 035/151] specify full DLL name instead of PYVER in GH Actions --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5c1c40f3..d720dee13 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,13 +12,36 @@ jobs: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: [3.6, 3.7, 3.8, 3.9] + pyver_minor: [6, 7, 8, 9] platform: [x64] shutdown_mode: [Normal, Soft] + include: + - os: ubuntu + pyver_minor: 6 + dll_suffix: m + - os: ubuntu + pyver_minor: 7 + dll_suffix: m + + - os: macos + dll_prefix: lib + dll_pyver_major: '3.' + - os: ubuntu + dll_prefix: lib + dll_pyver_major: '3.' + - os: windows + dll_pyver_major: '3' + + - os: ubuntu + dll_ext: .so + - os: windows + dll_ext: .dll + - os: macos + dll_ext: .dylib env: PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} - PYTHONNET_PYVER: ${{ matrix.PYTHON }} + PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} steps: - name: Set Environment on macOS @@ -33,10 +56,10 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v1 - - name: Set up Python ${{ matrix.python }} + - name: Set up Python 3.${{ matrix.pyver_minor }} uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python }} + python-version: 3.${{ matrix.pyver_minor }} architecture: ${{ matrix.platform }} - name: Install dependencies From b4cb37e06ed8a9e1ba6a9f206853f30da18c3bf6 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 21:04:24 -0800 Subject: [PATCH 036/151] use Microsoft.Net.Compilers.Toolset instead of Microsoft.Net.Compilers should fix Mac build --- Directory.Build.props | 2 +- src/runtime/Python.Runtime.csproj | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 4406a1870..cad319287 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,7 +8,7 @@ - + all runtime; build; native; contentfiles; analyzers diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index a09b0ff78..32a467dd8 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,6 @@ 9.0 Python.Runtime Python.Runtime - 9.0 pythonnet https://github.com/pythonnet/pythonnet/blob/master/LICENSE https://github.com/pythonnet/pythonnet From f68e581391e8dfc98397f417221b4e708c28a717 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 21:09:01 -0800 Subject: [PATCH 037/151] in CI MacOS python DLL has 'm' suffix --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d720dee13..fd304af78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,6 +26,7 @@ jobs: - os: macos dll_prefix: lib dll_pyver_major: '3.' + dll_suffix: m - os: ubuntu dll_prefix: lib dll_pyver_major: '3.' From cda604a29dd9672dce930b8c4d1ae77564169ffc Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 21:16:08 -0800 Subject: [PATCH 038/151] only set PYTHONNET_PYDLL for test runs from .NET --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd304af78..9c37f3941 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,7 +42,6 @@ jobs: env: PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} - PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} steps: - name: Set Environment on macOS @@ -78,10 +77,14 @@ jobs: - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython + env: + PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} - name: Python tests run from .NET run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ if: ${{ matrix.os == 'windows' }} # Not working for others right now + env: + PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} # TODO: Run perf tests # TODO: Run mono tests on Windows? From 398289294712c5e6a50c3f97301b425606077db9 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 21:38:41 -0800 Subject: [PATCH 039/151] workaround for pytest/clr module not preloading python C API library --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c37f3941..3c8fabcc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,8 +71,16 @@ jobs: python setup.py configure pip install -v . + # TODO this should be gone once clr module sets PythonDLL or preloads it - name: Python Tests run: pytest + if: ${{ matrix.os != 'macos' }} + env: + PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} + + - name: Python Tests + run: pytest + if: ${{ matrix.os == 'macos' }} - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ From a6cbe20cc81de4b0fc2b331b0a8ba53fc2d0311b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Mon, 25 Jan 2021 19:27:45 -0800 Subject: [PATCH 040/151] addressed a few code comments --- src/runtime/NewReference.cs | 10 +++++++++ src/runtime/converter.cs | 4 ++-- src/runtime/importhook.cs | 5 ++--- src/runtime/pyfloat.cs | 4 ++-- src/runtime/pyobject.cs | 2 +- src/runtime/runtime.cs | 42 +++++++++++-------------------------- src/runtime/runtime_data.cs | 2 +- src/runtime/typemanager.cs | 2 +- 8 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index b9d2602f7..f19dfd04c 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -11,6 +11,16 @@ ref struct NewReference { IntPtr pointer; + /// Creates a pointing to the same object + public NewReference(BorrowedReference reference, bool canBeNull = false) + { + var address = canBeNull + ? reference.DangerousGetAddressOrNull() + : reference.DangerousGetAddress(); + Runtime.XIncref(address); + this.pointer = address; + } + [Pure] public static implicit operator BorrowedReference(in NewReference reference) => new BorrowedReference(reference.pointer); diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 54124ad34..f3b378113 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -241,9 +241,9 @@ internal static IntPtr ToPython(object value, Type type) // return Runtime.PyFloat_FromDouble((double)((float)value)); string ss = ((float)value).ToString(nfi); IntPtr ps = Runtime.PyString_FromString(ss); - IntPtr op = Runtime.PyFloat_FromString(ps, IntPtr.Zero); + NewReference op = Runtime.PyFloat_FromString(new BorrowedReference(ps));; Runtime.XDecref(ps); - return op; + return op.DangerousMoveToPointerOrNull(); case TypeCode.Double: return Runtime.PyFloat_FromDouble((double)value); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 8d1a60342..066c765fe 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -306,15 +306,14 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) var mod = ManagedType.GetManagedObject(module) as ModuleObject; mod?.LoadNames(); } - return Runtime.NewRef(module).DangerousMoveToPointer(); + return new NewReference(module).DangerousMoveToPointer(); } if (clr_prefix != null) { return GetCLRModule(fromList).DangerousMoveToPointerOrNull(); } module = Runtime.PyDict_GetItemString(modules, names[0]); - return Runtime.NewRefOrNull(module) - .DangerousMoveToPointer(); + return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); } Exceptions.Clear(); diff --git a/src/runtime/pyfloat.cs b/src/runtime/pyfloat.cs index b07b95de1..a1752ff68 100644 --- a/src/runtime/pyfloat.cs +++ b/src/runtime/pyfloat.cs @@ -66,9 +66,9 @@ private static IntPtr FromString(string value) { using (var s = new PyString(value)) { - IntPtr val = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); + NewReference val = Runtime.PyFloat_FromString(s.Reference); PythonException.ThrowIfIsNull(val); - return val; + return val.DangerousMoveToPointerOrNull(); } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index b9125f40d..902d8c734 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -272,7 +272,7 @@ public bool HasAttr(PyObject name) { if (name == null) throw new ArgumentNullException(nameof(name)); - return Runtime.PyObject_HasAttr(obj, name.obj) != 0; + return Runtime.PyObject_HasAttr(Reference, name.Reference) != 0; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index f5d5cb371..990ac2a9f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -715,15 +715,6 @@ internal static unsafe void XIncref(IntPtr op) #endif } - internal static NewReference NewRef(BorrowedReference reference) - { - var address = reference.DangerousGetAddress(); - XIncref(address); - return NewReference.DangerousFromPointer(address); - } - internal static NewReference NewRefOrNull(BorrowedReference reference) - => reference.IsNull ? default : NewRef(reference); - /// /// Increase Python's ref counter for the given object, and get the object back. /// @@ -1039,19 +1030,12 @@ internal static int PyObject_HasAttrString(BorrowedReference pointer, string nam internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) { - IntPtr nameMem = Marshal.StringToHGlobalAnsi(name); - try - { - return Delegates.PyObject_GetAttrString(pointer, nameMem); - } - finally - { - Marshal.FreeHGlobal(nameMem); - } + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_GetAttrString(pointer, namePtr); } - internal static IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name) => Delegates.PyObject_GetAttrString(pointer, name); + internal static IntPtr PyObject_GetAttrString(IntPtr pointer, StrPtr name) => Delegates.PyObject_GetAttrString(pointer, name); internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) @@ -1060,7 +1044,7 @@ internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr v return Delegates.PyObject_SetAttrString(pointer, namePtr, value); } - internal static int PyObject_HasAttr(IntPtr pointer, IntPtr name) => Delegates.PyObject_HasAttr(pointer, name); + internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) @@ -1324,7 +1308,7 @@ internal static bool PyFloat_Check(IntPtr ob) internal static IntPtr PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); - internal static IntPtr PyFloat_FromString(IntPtr value, IntPtr junk) => Delegates.PyFloat_FromString(value, junk); + internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); internal static double PyFloat_AsDouble(IntPtr ob) => Delegates.PyFloat_AsDouble(ob); @@ -1990,8 +1974,6 @@ internal static int PySys_SetObject(string name, BorrowedReference ob) //==================================================================== // Python type object API //==================================================================== - static readonly delegate* unmanaged[Cdecl] pyType_Check; - internal static bool PyType_Check(IntPtr ob) { return PyObject_TypeCheck(ob, PyTypeType); @@ -2272,7 +2254,7 @@ private static class Delegates static Delegates() { PyDictProxy_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDictProxy_New), GetUnmanagedDll(_PythonDll)); -Py_IncRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IncRef), GetUnmanagedDll(_PythonDll)); + Py_IncRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IncRef), GetUnmanagedDll(_PythonDll)); Py_DecRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_DecRef), GetUnmanagedDll(_PythonDll)); Py_Initialize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Initialize), GetUnmanagedDll(_PythonDll)); Py_InitializeEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_InitializeEx), GetUnmanagedDll(_PythonDll)); @@ -2322,9 +2304,9 @@ static Delegates() PyCFunction_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_Call), GetUnmanagedDll(_PythonDll)); PyMethod_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_New), GetUnmanagedDll(_PythonDll)); PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); - PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); - PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); PyObject_GetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttr), GetUnmanagedDll(_PythonDll)); PyObject_SetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttr), GetUnmanagedDll(_PythonDll)); PyObject_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetItem), GetUnmanagedDll(_PythonDll)); @@ -2382,7 +2364,7 @@ static Delegates() PyLong_FromVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromVoidPtr), GetUnmanagedDll(_PythonDll)); PyLong_AsVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsVoidPtr), GetUnmanagedDll(_PythonDll)); PyFloat_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromDouble), GetUnmanagedDll(_PythonDll)); - PyFloat_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromString), GetUnmanagedDll(_PythonDll)); + PyFloat_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromString), GetUnmanagedDll(_PythonDll)); PyFloat_AsDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_AsDouble), GetUnmanagedDll(_PythonDll)); PyNumber_Add = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Add), GetUnmanagedDll(_PythonDll)); PyNumber_Subtract = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Subtract), GetUnmanagedDll(_PythonDll)); @@ -2594,9 +2576,9 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyCFunction_Call { get; } internal static delegate* unmanaged[Cdecl] PyMethod_New { get; } internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } - internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } - internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_SetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetItem { get; } @@ -2647,7 +2629,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyLong_FromVoidPtr { get; } internal static delegate* unmanaged[Cdecl] PyLong_AsVoidPtr { get; } internal static delegate* unmanaged[Cdecl] PyFloat_FromDouble { get; } - internal static delegate* unmanaged[Cdecl] PyFloat_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_FromString { get; } internal static delegate* unmanaged[Cdecl] PyFloat_AsDouble { get; } internal static delegate* unmanaged[Cdecl] PyNumber_Add { get; } internal static delegate* unmanaged[Cdecl] PyNumber_Subtract { get; } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index abf057cef..0b3bf3017 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -85,7 +85,7 @@ internal static void Stash() Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); ClearCLRData(); - #warning this leaks memory in mem + NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); PySys_SetObject("clr_data", capsule); // Let the dictionary own the reference diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3a107d53d..e0564b243 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -938,7 +938,7 @@ public static IntPtr CreateObjectType() resRef.Dispose(); BorrowedReference A = Runtime.PyDict_GetItemString(globals, "A"); Debug.Assert(!A.IsNull); - return Runtime.NewRef(A).DangerousMoveToPointer(); + return new NewReference(A).DangerousMoveToPointer(); } } } From 1afae4cf13962009919844df87cd690465ae9a51 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 29 Jan 2021 07:50:08 +0100 Subject: [PATCH 041/151] Fix exception string (#1360) Fix exception stacktrace formatting Ensure that the stacktrace list is joined into a single string and adjust the formatting slightly such that the .NET stacktrace is separated from the one from Python. The Python traceback is also reversed to match the order used in .NET. Fixes #1359. --- CHANGELOG.md | 33 +++++++++++++------------- src/embed_tests/TestPythonException.cs | 16 ++++++++++++- src/runtime/pythonexception.cs | 33 +++++++++++++++----------- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64f3cdef..5f2e544df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,22 +39,23 @@ or the DLL must be loaded in advance. This must be done before calling any other ### Fixed -- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash -- Fix incorrect dereference in params array handling -- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) -- Fix `object[]` parameters taking precedence when should not in overload resolution -- Fixed a bug where all .NET class instances were considered Iterable -- Fix incorrect choice of method to invoke when using keyword arguments. -- Fix non-delegate types incorrectly appearing as callable. -- Indexers can now be used with interface objects -- Fixed a bug where indexers could not be used if they were inherited -- Made it possible to use `__len__` also on `ICollection<>` interface objects -- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions -- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects -- Fixed objects returned by enumerating `PyObject` being disposed too soon -- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException ([#1325][i1325]) -- `import` may now raise errors with more detail than "No module named X" -- Providing an invalid type parameter to a generic type or method produces a helpful Python error +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling +- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Fixed a bug where all .NET class instances were considered Iterable +- Fix incorrect choice of method to invoke when using keyword arguments. +- Fix non-delegate types incorrectly appearing as callable. +- Indexers can now be used with interface objects +- Fixed a bug where indexers could not be used if they were inherited +- Made it possible to use `__len__` also on `ICollection<>` interface objects +- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions +- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects +- Fixed objects returned by enumerating `PyObject` being disposed too soon +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException +- `import` may now raise errors with more detail than "No module named X" +- Exception stacktraces on `PythonException.StackTrace` are now properly formatted +- Providing an invalid type parameter to a generic type or method produces a helpful Python error ### Removed diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index e51da4d4f..31addfba1 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -65,7 +65,21 @@ public void TestPythonExceptionFormat() } catch (PythonException ex) { - Assert.That(ex.Format(), Does.Contain("Traceback").And.Contains("(most recent call last):").And.Contains("ValueError: Error!")); + // Console.WriteLine($"Format: {ex.Format()}"); + // Console.WriteLine($"Stacktrace: {ex.StackTrace}"); + Assert.That( + ex.Format(), + Does.Contain("Traceback") + .And.Contains("(most recent call last):") + .And.Contains("ValueError: Error!") + ); + + // Check that the stacktrace is properly formatted + Assert.That( + ex.StackTrace, + Does.Not.StartWith("[") + .And.Not.Contain("\\n") + ); } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ad4d79960..7dd4f0811 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -13,9 +14,9 @@ public class PythonException : System.Exception, IDisposable private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; private IntPtr _pyTB = IntPtr.Zero; - private string _tb = ""; - private string _message = ""; - private string _pythonTypeName = ""; + private readonly string _tb = ""; + private readonly string _message = ""; + private readonly string _pythonTypeName = ""; private bool disposed = false; private bool _finalized = false; @@ -44,16 +45,23 @@ public PythonException() } _message = type + " : " + message; } + if (_pyTB != IntPtr.Zero) { - using (PyObject tb_module = PythonEngine.ImportModule("traceback")) - { - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) - { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); - } + using PyObject tb_module = PythonEngine.ImportModule("traceback"); + + Runtime.XIncref(_pyTB); + using var pyTB = new PyObject(_pyTB); + + using var tbList = tb_module.InvokeMethod("format_tb", pyTB); + + var sb = new StringBuilder(); + // Reverse Python's traceback list to match the order used in C# + // stacktraces + foreach (var line in tbList.Reverse()) { + sb.Append(line.ToString()); } + _tb = sb.ToString(); } PythonEngine.ReleaseLock(gs); } @@ -136,10 +144,7 @@ public override string Message /// /// A string representing the python exception stack trace. /// - public override string StackTrace - { - get { return _tb + base.StackTrace; } - } + public override string StackTrace => $"{_tb}===\n{base.StackTrace}"; /// /// Python error type name. From d86bf3ceba5f54c8cdfc34026d976bcd19351f33 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 4 Feb 2021 21:33:19 +0100 Subject: [PATCH 042/151] ParameterInfo.Name needs to be checked for null before usage (#1375) This occured in trying to use F# code from Python. As the `.Name` property returns `null`, `ContainsKey` fails. Related documentation: https://docs.microsoft.com/en-us/dotnet/api/system.reflection.parameterinfo.name --- CHANGELOG.md | 1 + src/runtime/methodbinder.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f2e544df..d30dfa059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ or the DLL must be loaded in advance. This must be done before calling any other - `import` may now raise errors with more detail than "No module named X" - Exception stacktraces on `PythonException.StackTrace` are now properly formatted - Providing an invalid type parameter to a generic type or method produces a helpful Python error +- Empty parameter names (as can be generated from F#) do not cause crashes ### Removed diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 034c1c3e8..8f74e0052 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -622,7 +622,7 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) { var parameter = pi[paramIndex]; - bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + bool hasNamedParam = parameter.Name != null ? kwargDict.ContainsKey(parameter.Name) : false; bool isNewReference = false; if (paramIndex >= pyArgCount && !(hasNamedParam || (paramsArray && paramIndex == arrayStart))) From 6b2347a187fc17b6ce4b60c0a5f3f39150bf20c1 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 12 Feb 2021 13:36:08 -0800 Subject: [PATCH 043/151] monthly NuGet release previews (#1381) --- .github/workflows/nuget-preview.yml | 60 +++++++++++++++++++++++++++++ Directory.Build.props | 1 + LICENSE | 2 +- pythonnet.sln | 20 ++++------ src/runtime/Python.Runtime.csproj | 17 +++++++- 5 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/nuget-preview.yml diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml new file mode 100644 index 000000000..951529452 --- /dev/null +++ b/.github/workflows/nuget-preview.yml @@ -0,0 +1,60 @@ +name: GitHub Actions + +on: + schedule: + - cron: "5 4 3 */1 *" # once a month, at 4:05 on 3rd + +jobs: + release: + name: Release Preview + runs-on: ubuntu-latest + environment: NuGet + timeout-minutes: 10 + + env: + PYTHONNET_SHUTDOWN_MODE: Normal + + steps: + - name: Get Date + run: | + echo "DATE_VER=$(date "+%Y-%m-%d")" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + + - name: Build and Install + run: | + python setup.py configure + pip install -v . + + - name: Python Tests + run: pytest + env: + PYTHONNET_PYDLL: libpython3.8.so + + - name: Embedding tests + run: dotnet test --runtime any-ubuntu src/embed_tests/ + env: + PYTHONNET_PYDLL: libpython3.8.so + + - name: Pack + run: dotnet pack --configuration Release --version-suffix preview${{env.DATE_VER}} --output "Release-Preview" + + - name: Publish NuGet + run: dotnet nuget push --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_MONTHLY }} Release-Preview/*.nupkg + + # TODO: Run perf tests + # TODO: Run mono tests on Windows? diff --git a/Directory.Build.props b/Directory.Build.props index cad319287..55d091b4e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -5,6 +5,7 @@ pythonnet Python.NET 7.3 + false diff --git a/LICENSE b/LICENSE index 19c31a12f..f3a638346 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2006-2020 the contributors of the Python.NET project +Copyright (c) 2006-2021 the contributors of the Python.NET project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), diff --git a/pythonnet.sln b/pythonnet.sln index 4da4d7e99..c80ee8e60 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -21,6 +21,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F .editorconfig = .editorconfig .gitignore = .gitignore CHANGELOG.md = CHANGELOG.md + LICENSE = LICENSE README.rst = README.rst EndProjectSection EndProject @@ -30,6 +31,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF- ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 .github\workflows\main.yml = .github\workflows\main.yml + .github\workflows\nuget-preview.yml = .github\workflows\nuget-preview.yml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" @@ -51,6 +53,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{BC426F42 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PythonTestsRunner", "src\python_tests_runner\Python.PythonTestsRunner.csproj", "{35CBBDEB-FC07-4D04-9D3E-F88FC180110B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{142A6752-C2C2-4F95-B982-193418001B65}" + ProjectSection(SolutionItems) = preProject + configured.props = configured.props + Directory.Build.props = Directory.Build.props + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -145,18 +153,6 @@ Global {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.Build.0 = Release|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.Build.0 = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x64.ActiveCfg = Debug|Any CPU diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 32a467dd8..aa2ea4f91 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,21 +5,35 @@ 9.0 Python.Runtime Python.Runtime + pythonnet - https://github.com/pythonnet/pythonnet/blob/master/LICENSE + LICENSE https://github.com/pythonnet/pythonnet git python interop dynamic dlr Mono pinvoke https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico https://pythonnet.github.io/ + true + Python and CLR (.NET and Mono) cross-platform language interop + + true + true + snupkg + 1591;NU1701 True + + true $(DefineConstants);$(ConfiguredConstants) + + + + @@ -30,5 +44,6 @@ + From 4da19543c8651540f601c44c560c6530c12820f7 Mon Sep 17 00:00:00 2001 From: gpetrou <4172445+gpetrou@users.noreply.github.com> Date: Sat, 13 Feb 2021 12:33:50 +0000 Subject: [PATCH 044/151] Sign Runtime DLL with a strong name (#1382) --- CHANGELOG.md | 1 + src/embed_tests/Python.EmbeddingTest.csproj | 2 ++ src/runtime/Properties/AssemblyInfo.cs | 2 +- src/runtime/Python.Runtime.csproj | 7 +++++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d30dfa059..1a1fd340e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ when .NET expects an integer [#1342][i1342] - BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. - BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. +- Sign Runtime DLL with a strong name ### Fixed diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 1bb4fed11..67a7d3338 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -2,6 +2,8 @@ net472;netcoreapp3.1 + ..\pythonnet.snk + true diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 470488c02..3417bccc8 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -1,3 +1,3 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Python.EmbeddingTest")] +[assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index aa2ea4f91..ef530d69a 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -19,7 +19,10 @@ true true snupkg - + + ..\pythonnet.snk + true + 1591;NU1701 True @@ -31,7 +34,7 @@ - + From b92d9295c3d7d946a5302fd1072bef92b5388185 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 13:10:44 +0100 Subject: [PATCH 045/151] Replace clr module by loading through clr_loader clr_loader is based on CFFI and allows loading pythonnet on different .NET runtime implementations, currently .NET Framework, .NET Core and Mono. Apart from dropping the respective old code, this requires the following changes: - Move libpython discovery into Python by vendoring and adjusting `find_libpython` - Adjust the GIL handling in the startup code as CFFI releases the GIL when calling the external function - Remove the intermittent `configure` command as it is not required anymore - Adjust a few test-cases - Remove `_AtExit` Due to issues with the reference counts, the `atexit` callback is currently essentially a no-op until these are fixed. --- .github/workflows/main.yml | 53 +-- .gitignore | 3 - Directory.Build.props | 1 - appveyor.yml | 3 +- clr.py | 39 +- pythonnet.sln | 2 - pythonnet/__init__.py | 74 +++- pythonnet/find_libpython/__init__.py | 399 ++++++++++++++++++++ pythonnet/find_libpython/__main__.py | 2 + pythonnet/mono/.gitkeep | 0 pythonnet/netfx/.gitkeep | 0 pythonnet/util/__init__.py | 1 + requirements.txt | 2 +- setup.py | 125 +------ src/clrmodule/ClrModule.cs | 113 ------ src/clrmodule/Properties/AssemblyInfo.cs | 5 - src/clrmodule/clrmodule.csproj | 24 -- src/embed_tests/TestNativeTypeOffset.cs | 7 +- src/monoclr/clrmod.c | 215 ----------- src/runtime/Python.Runtime.csproj | 3 +- src/runtime/loader.cs | 83 +++++ src/runtime/moduleobject.cs | 6 - src/runtime/native/TypeOffset.cs | 1 - src/runtime/pythonengine.cs | 14 +- src/runtime/runtime.cs | 441 +++++++++++------------ src/tests/test_method.py | 3 - src/tests/test_sysargv.py | 5 +- tools/geninterop/geninterop.py | 4 +- 28 files changed, 804 insertions(+), 824 deletions(-) create mode 100644 pythonnet/find_libpython/__init__.py create mode 100644 pythonnet/find_libpython/__main__.py delete mode 100644 pythonnet/mono/.gitkeep delete mode 100644 pythonnet/netfx/.gitkeep create mode 100644 pythonnet/util/__init__.py delete mode 100644 src/clrmodule/ClrModule.cs delete mode 100644 src/clrmodule/Properties/AssemblyInfo.cs delete mode 100644 src/clrmodule/clrmodule.csproj delete mode 100644 src/monoclr/clrmod.c create mode 100644 src/runtime/loader.cs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3c8fabcc1..10959ea4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,33 +12,9 @@ jobs: fail-fast: false matrix: os: [windows, ubuntu, macos] - pyver_minor: [6, 7, 8, 9] + python: ["3.6", "3.7", "3.8", "3.9"] platform: [x64] shutdown_mode: [Normal, Soft] - include: - - os: ubuntu - pyver_minor: 6 - dll_suffix: m - - os: ubuntu - pyver_minor: 7 - dll_suffix: m - - - os: macos - dll_prefix: lib - dll_pyver_major: '3.' - dll_suffix: m - - os: ubuntu - dll_prefix: lib - dll_pyver_major: '3.' - - os: windows - dll_pyver_major: '3' - - - os: ubuntu - dll_ext: .so - - os: windows - dll_ext: .dll - - os: macos - dll_ext: .dylib env: PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} @@ -56,10 +32,10 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v1 - - name: Set up Python 3.${{ matrix.pyver_minor }} + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v2 with: - python-version: 3.${{ matrix.pyver_minor }} + python-version: ${{ matrix.python }} architecture: ${{ matrix.platform }} - name: Install dependencies @@ -68,31 +44,26 @@ jobs: - name: Build and Install run: | - python setup.py configure pip install -v . - # TODO this should be gone once clr module sets PythonDLL or preloads it - - name: Python Tests - run: pytest - if: ${{ matrix.os != 'macos' }} - env: - PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} + - name: Set Python DLL path (non Windows) + if: ${{ matrix.os != 'windows' }} + run: | + python -m pythonnet.find_libpython --export >> $GITHUB_ENV + - name: Set Python DLL path (Windows) + if: ${{ matrix.os == 'windows' }} + run: | + python -m pythonnet.find_libpython --export | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Python Tests run: pytest - if: ${{ matrix.os == 'macos' }} - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ - if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython - env: - PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} - name: Python tests run from .NET run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ - if: ${{ matrix.os == 'windows' }} # Not working for others right now - env: - PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}} # TODO: Run perf tests # TODO: Run mono tests on Windows? diff --git a/.gitignore b/.gitignore index 673681317..cdb152157 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ /src/runtime/interopNative.cs -# Configuration data -configured.props - # General binaries and Build results *.dll *.exe diff --git a/Directory.Build.props b/Directory.Build.props index 55d091b4e..edc8ba513 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -18,5 +18,4 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/appveyor.yml b/appveyor.yml index 21e816f38..cc3815c62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,9 +30,9 @@ environment: init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% - - set PYTHONNET_PYDLL=python%PY_VER%.dll - set PYTHON=C:\PYTHON%PY_VER% - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64) + - set PYTHONNET_PYDLL=%PYTHON%\python%PY_VER%.dll # Put desired Python version first in PATH - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% @@ -42,7 +42,6 @@ install: - pip install --upgrade -r requirements.txt --quiet build_script: - - python setup.py configure # Create clean `sdist`. Only used for releases - python setup.py --quiet sdist - python setup.py bdist_wheel diff --git a/clr.py b/clr.py index 711333dd2..20a975f96 100644 --- a/clr.py +++ b/clr.py @@ -2,40 +2,5 @@ Legacy Python.NET loader for backwards compatibility """ -def _get_netfx_path(): - import os, sys - - if sys.maxsize > 2 ** 32: - arch = "amd64" - else: - arch = "x86" - - return os.path.join(os.path.dirname(__file__), "pythonnet", "netfx", arch, "clr.pyd") - - -def _get_mono_path(): - import os, glob - - paths = glob.glob(os.path.join(os.path.dirname(__file__), "pythonnet", "mono", "clr.*so")) - return paths[0] - - -def _load_clr(): - import sys - from importlib import util - - if sys.platform == "win32": - path = _get_netfx_path() - else: - path = _get_mono_path() - - del sys.modules[__name__] - - spec = util.spec_from_file_location("clr", path) - clr = util.module_from_spec(spec) - spec.loader.exec_module(clr) - - sys.modules[__name__] = clr - - -_load_clr() +from pythonnet import load +load() diff --git a/pythonnet.sln b/pythonnet.sln index c80ee8e60..30f4fd344 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -6,8 +6,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime", "src\runti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{819E089B-4770-400E-93C6-4F7A35F0EA12}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\Python.Test.csproj", "{14EF9518-5BB7-4F83-8686-015BD2CC788E}" diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 5c197e146..692fd5700 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -1,3 +1,71 @@ -def get_assembly_path(): - import os - return os.path.dirname(__file__) + "/runtime/Python.Runtime.dll" +import os +import sys +import clr_loader + +_RUNTIME = None +_LOADER_ASSEMBLY = None +_FFI = None +_LOADED = False + + +def set_runtime(runtime): + global _RUNTIME + if _LOADED: + raise RuntimeError("The runtime {runtime} has already been loaded".format(_RUNTIME)) + + _RUNTIME = runtime + + +def set_default_runtime() -> None: + if sys.platform == 'win32': + set_runtime(clr_loader.get_netfx()) + else: + set_runtime(clr_loader.get_mono()) + + +def load(): + global _FFI, _LOADED, _LOADER_ASSEMBLY + + if _LOADED: + return + + from .find_libpython import linked_libpython + from os.path import join, dirname + + if _RUNTIME is None: + # TODO: Warn, in the future the runtime must be set explicitly, either + # as a config/env variable or via set_runtime + set_default_runtime() + + dll_path = join(dirname(__file__), "runtime", "Python.Runtime.dll") + libpython = linked_libpython() + + if libpython and _FFI is None and sys.platform != "win32": + # Load and leak libpython handle s.t. the .NET runtime doesn't dlcloses + # it + import posix + + import cffi + _FFI = cffi.FFI() + _FFI.dlopen(libpython, posix.RTLD_NODELETE | posix.RTLD_LOCAL) + + _LOADER_ASSEMBLY = _RUNTIME.get_assembly(dll_path) + + func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Initialize"] + if func(f"{libpython or ''}".encode("utf8")) != 0: + raise RuntimeError("Failed to initialize Python.Runtime.dll") + + import atexit + atexit.register(unload) + + +def unload(): + global _RUNTIME + if _LOADER_ASSEMBLY is not None: + func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Shutdown"] + if func(b"") != 0: + raise RuntimeError("Failed to call Python.NET shutdown") + + if _RUNTIME is not None: + # TODO: Add explicit `close` to clr_loader + _RUNTIME = None diff --git a/pythonnet/find_libpython/__init__.py b/pythonnet/find_libpython/__init__.py new file mode 100644 index 000000000..185540c8f --- /dev/null +++ b/pythonnet/find_libpython/__init__.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +# License +# +# Copyright 2018, Takafumi Arakaki +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, absolute_import + +from logging import getLogger +import ctypes.util +import functools +import os +import sys +import sysconfig + +logger = getLogger("find_libpython") + +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") +if SHLIB_SUFFIX is None: + if is_windows: + SHLIB_SUFFIX = ".dll" + else: + SHLIB_SUFFIX = ".so" +if is_apple: + # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. + # Let's not use the value from sysconfig. + SHLIB_SUFFIX = ".dylib" + + +def linked_libpython(): + """ + Find the linked libpython using dladdr (in *nix). + + Returns + ------- + path : str or None + A path to linked libpython. Return `None` if statically linked. + """ + if is_windows: + return _linked_libpython_windows() + return _linked_libpython_unix() + + +class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + +def _linked_libpython_unix(): + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + + dlinfo = Dl_info() + retcode = libdl.dladdr( + ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), + ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + path = os.path.realpath(dlinfo.dli_fname.decode()) + if path == os.path.realpath(sys.executable): + return None + return path + + +def _linked_libpython_windows(): + """ + Based on: https://stackoverflow.com/a/16659821 + """ + from ctypes.wintypes import HANDLE, LPWSTR, DWORD + + GetModuleFileName = ctypes.windll.kernel32.GetModuleFileNameW + GetModuleFileName.argtypes = [HANDLE, LPWSTR, DWORD] + GetModuleFileName.restype = DWORD + + MAX_PATH = 260 + try: + buf = ctypes.create_unicode_buffer(MAX_PATH) + GetModuleFileName(ctypes.pythonapi._handle, buf, MAX_PATH) + return buf.value + except (ValueError, OSError): + return None + + + +def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows and name.startswith("lib"): + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def uniquifying(items): + """ + Yield items while excluding the duplicates and preserving the order. + + >>> list(uniquifying([1, 2, 1, 2, 3])) + [1, 2, 3] + """ + seen = set() + for x in items: + if x not in seen: + yield x + seen.add(x) + + +def uniquified(func): + """ Wrap iterator returned from `func` by `uniquifying`. """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return uniquifying(func(*args, **kwds)) + return wrapper + + +@uniquified +def candidate_names(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate file names of libpython. + + Yields + ------ + name : str + Candidate name libpython. + """ + LDLIBRARY = sysconfig.get_config_var("LDLIBRARY") + if LDLIBRARY and not LDLIBRARY.endswith(".a"): + yield LDLIBRARY + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY and not LIBRARY.endswith(".a"): + yield os.path.splitext(LIBRARY)[0] + suffix + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + # VERSION is X.Y in Linux/macOS and XY in Windows: + VERSION=(sysconfig.get_python_version() or + "{v.major}.{v.minor}".format(v=sys.version_info) or + sysconfig.get_config_var("VERSION")), + ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + + for stem in [ + "python{VERSION}{ABIFLAGS}".format(**sysdata), + "python{VERSION}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]: + yield dlprefix + stem + suffix + + + +@uniquified +def candidate_paths(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + + yield linked_libpython() + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var('LIBPL')) + append_truthy(lib_dirs, sysconfig.get_config_var('srcdir')) + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + # LIBPL seems to be the right config_var to use. It is the one + # used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.7.0/Misc/python-config.in#L55-L57 + # + # But we try other places just in case. + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + lib_basenames = list(candidate_names(suffix=suffix)) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + +# Possibly useful links: +# * https://packages.ubuntu.com/bionic/amd64/libpython3.6/filelist +# * https://github.com/Valloric/ycmd/issues/518 +# * https://github.com/Valloric/ycmd/pull/519 + + +def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + if is_apple: + return normalize_path(_remove_suffix_apple(path), + suffix=".so", is_apple=False) + return None + + +def _remove_suffix_apple(path): + """ + Strip off .so or .dylib. + + >>> _remove_suffix_apple("libpython.so") + 'libpython' + >>> _remove_suffix_apple("libpython.dylib") + 'libpython' + >>> _remove_suffix_apple("libpython3.7") + 'libpython3.7' + """ + if path.endswith(".dylib"): + return path[:-len(".dylib")] + if path.endswith(".so"): + return path[:-len(".so")] + return path + + +@uniquified +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. + + Yields + ------ + path : str + Existing path to a libpython. + """ + logger.debug("is_windows = %s", is_windows) + logger.debug("is_apple = %s", is_apple) + for path in candidate_paths(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + else: + logger.debug("Not found.") + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def print_all(items): + for x in items: + print(x) + + +def cli_find_libpython(cli_op, verbose, export): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig( + format="%(levelname)s %(message)s", + level=logging.DEBUG) + + if cli_op == "list-all": + print_all(finding_libpython()) + elif cli_op == "candidate-names": + print_all(candidate_names()) + elif cli_op == "candidate-paths": + print_all(p for p in candidate_paths() if p and os.path.isabs(p)) + else: + path = find_libpython() + if path is None: + return 1 + if export: + print(f"PYTHONNET_PYDLL={path}") + else: + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list-all", + action="store_const", dest="cli_op", const="list-all", + help="Print list of all paths found.") + group.add_argument( + "--candidate-names", + action="store_const", dest="cli_op", const="candidate-names", + help="Print list of candidate names of libpython.") + group.add_argument( + "--candidate-paths", + action="store_const", dest="cli_op", const="candidate-paths", + help="Print list of candidate paths of libpython.") + group.add_argument( + "--export", + action="store_true", + help="Print as an environment export expression" + ) + + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) diff --git a/pythonnet/find_libpython/__main__.py b/pythonnet/find_libpython/__main__.py new file mode 100644 index 000000000..031df43e1 --- /dev/null +++ b/pythonnet/find_libpython/__main__.py @@ -0,0 +1,2 @@ +from . import main +main() diff --git a/pythonnet/mono/.gitkeep b/pythonnet/mono/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/pythonnet/netfx/.gitkeep b/pythonnet/netfx/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/pythonnet/util/__init__.py b/pythonnet/util/__init__.py new file mode 100644 index 000000000..75d4bad8c --- /dev/null +++ b/pythonnet/util/__init__.py @@ -0,0 +1 @@ +from .find_libpython import find_libpython diff --git a/requirements.txt b/requirements.txt index 6f25858bc..f5aedfc3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ psutil coverage codecov -# Platform specific requirements wheel pycparser setuptools +clr-loader diff --git a/setup.py b/setup.py index 06a26ef95..ffa18902e 100644 --- a/setup.py +++ b/setup.py @@ -8,79 +8,9 @@ import sys, os -BUILD_MONO = True -BUILD_NETFX = True - PY_MAJOR = sys.version_info[0] PY_MINOR = sys.version_info[1] -CONFIGURED_PROPS = "configured.props" - - -def _get_interop_filename(): - """interopXX.cs is auto-generated as part of the build. - For common windows platforms pre-generated files are included - as most windows users won't have Clang installed, which is - required to generate the file. - """ - interop_filename = "interop{0}{1}{2}.cs".format( - PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "") - ) - return os.path.join("src", "runtime", interop_filename) - - -# Write configuration to configured.props. This will go away once all of these -# can be decided at runtime. -def _write_configure_props(): - defines = [ - "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR), - ] - - if sys.platform == "win32": - defines.append("WINDOWS") - - if hasattr(sys, "abiflags"): - if "d" in sys.abiflags: - defines.append("PYTHON_WITH_PYDEBUG") - if "m" in sys.abiflags: - defines.append("PYTHON_WITH_PYMALLOC") - - # check the interop file exists, and create it if it doesn't - interop_file = _get_interop_filename() - if not os.path.exists(interop_file): - print("Creating {0}".format(interop_file)) - geninterop = os.path.join("tools", "geninterop", "geninterop.py") - check_call([sys.executable, geninterop, interop_file]) - - import xml.etree.ElementTree as ET - - proj = ET.Element("Project") - props = ET.SubElement(proj, "PropertyGroup") - f = ET.SubElement(props, "PythonInteropFile") - f.text = os.path.basename(interop_file) - - c = ET.SubElement(props, "ConfiguredConstants") - c.text = " ".join(defines) - - ET.ElementTree(proj).write(CONFIGURED_PROPS) - - -class configure(Command): - """Configure command""" - - description = "Configure the pythonnet build" - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - self.announce("Writing configured.props...", level=distutils.log.INFO) - _write_configure_props() - class DotnetLib: def __init__(self, name, path, **kwargs): @@ -121,7 +51,6 @@ def finalize_options(self): def run(self): dotnet_modules = self.distribution.dotnet_libs - self.run_command("configure") for lib in dotnet_modules: output = os.path.join( @@ -188,7 +117,6 @@ def install_for_development(self): cmdclass = { "build": build, "build_dotnet": build_dotnet, - "configure": configure, "develop": develop, } @@ -204,54 +132,6 @@ def install_for_development(self): ) ] -if BUILD_NETFX: - dotnet_libs.extend( - [ - DotnetLib( - "clrmodule-amd64", - "src/clrmodule/", - runtime="win-x64", - output="pythonnet/netfx/amd64", - rename={"clr.dll": "clr.pyd"}, - ), - DotnetLib( - "clrmodule-x86", - "src/clrmodule/", - runtime="win-x86", - output="pythonnet/netfx/x86", - rename={"clr.dll": "clr.pyd"}, - ), - ] - ) - -ext_modules = [] - -if BUILD_MONO: - try: - mono_libs = check_output( - "pkg-config --libs mono-2", shell=True, encoding="utf8" - ) - mono_cflags = check_output( - "pkg-config --cflags mono-2", shell=True, encoding="utf8" - ) - cflags = mono_cflags.strip() - libs = mono_libs.strip() - - # build the clr python module - clr_ext = Extension( - "pythonnet.mono.clr", - language="c++", - sources=["src/monoclr/clrmod.c"], - extra_compile_args=cflags.split(" "), - extra_link_args=libs.split(" "), - ) - ext_modules.append(clr_ext) - except Exception: - print( - "Failed to find mono libraries via pkg-config, skipping the Mono CLR loader" - ) - - setup( cmdclass=cmdclass, name="pythonnet", @@ -261,12 +141,11 @@ def install_for_development(self): license="MIT", author="The Contributors of the Python.NET Project", author_email="pythonnet@python.org", - packages=["pythonnet"], - install_requires=["pycparser"], + packages=["pythonnet", "pythonnet.find_libpython"], + install_requires=["pycparser", "clr_loader"], long_description=long_description, # data_files=[("{install_platlib}", ["{build_lib}/pythonnet"])], py_modules=["clr"], - ext_modules=ext_modules, dotnet_libs=dotnet_libs, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs deleted file mode 100644 index ab0f6da9f..000000000 --- a/src/clrmodule/ClrModule.cs +++ /dev/null @@ -1,113 +0,0 @@ -//============================================================================ -// This file replaces the hand-maintained stub that used to implement clr.dll. -// This is a line-by-line port from IL back to C#. -// We now use RGiesecke.DllExport on the required static init method so it can be -// loaded by a standard CPython interpreter as an extension module. When it -// is loaded, it bootstraps the managed runtime integration layer and defers -// to it to do initialization and put the clr module into sys.modules, etc. - -// The "USE_PYTHON_RUNTIME_*" defines control what extra evidence is used -// to help the CLR find the appropriate Python.Runtime assembly. - -// If defined, the "pythonRuntimeVersionString" variable must be set to -// Python.Runtime's current version. -#define USE_PYTHON_RUNTIME_VERSION - -// If defined, the "PythonRuntimePublicKeyTokenData" data array must be -// set to Python.Runtime's public key token. (sn -T Python.Runtin.dll) -#define USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - -// If DEBUG is defined in the Build Properties, a few Console.WriteLine -// calls are made to indicate what's going on during the load... -//============================================================================ -using System; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Reflection; -using System.Runtime.InteropServices; -using NXPorts.Attributes; - -public class clrModule -{ - [DllExport("PyInit_clr", CallingConvention.StdCall)] - public static IntPtr PyInit_clr() - { - DebugPrint("Attempting to load 'Python.Runtime' using standard binding rules."); -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - var pythonRuntimePublicKeyTokenData = new byte[] { 0x50, 0x00, 0xfe, 0xa6, 0xcb, 0xa7, 0x02, 0xdd }; -#endif - - // Attempt to find and load Python.Runtime using standard assembly binding rules. - // This roughly translates into looking in order: - // - GAC - // - ApplicationBase - // - A PrivateBinPath under ApplicationBase - // With an unsigned assembly, the GAC is skipped. - var pythonRuntimeName = new AssemblyName("Python.Runtime") - { -#if USE_PYTHON_RUNTIME_VERSION - // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.5.0"), -#endif - CultureInfo = CultureInfo.InvariantCulture - }; -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - pythonRuntimeName.SetPublicKeyToken(pythonRuntimePublicKeyTokenData); -#endif - // We've got the AssemblyName with optional features; try to load it. - Assembly pythonRuntime; - try - { - pythonRuntime = Assembly.Load(pythonRuntimeName); - DebugPrint("Success loading 'Python.Runtime' using standard binding rules."); - } - catch (IOException ex) - { - DebugPrint($"'Python.Runtime' not found using standard binding rules: {ex}"); - try - { - // If the above fails for any reason, we fallback to attempting to load "Python.Runtime.dll" - // from the directory this assembly is running in. "This assembly" is probably "clr.pyd", - // sitting somewhere in PYTHONPATH. This is using Assembly.LoadFrom, and inherits all the - // caveats of that call. See MSDN docs for details. - // Suzanne Cook's blog is also an excellent source of info on this: - // http://blogs.msdn.com/suzcook/ - // http://blogs.msdn.com/suzcook/archive/2003/05/29/57143.aspx - // http://blogs.msdn.com/suzcook/archive/2003/06/13/57180.aspx - - Assembly executingAssembly = Assembly.GetExecutingAssembly(); - string assemblyDirectory = Path.GetDirectoryName(executingAssembly.Location); - if (assemblyDirectory == null) - { - throw new InvalidOperationException(executingAssembly.Location); - } - string pythonRuntimeDllPath = Path.Combine(assemblyDirectory, "Python.Runtime.dll"); - DebugPrint($"Attempting to load Python.Runtime from: '{pythonRuntimeDllPath}'."); - pythonRuntime = Assembly.LoadFrom(pythonRuntimeDllPath); - DebugPrint($"Success loading 'Python.Runtime' from: '{pythonRuntimeDllPath}'."); - } - catch (InvalidOperationException) - { - DebugPrint("Could not load 'Python.Runtime'."); - return IntPtr.Zero; - } - } - - // Once here, we've successfully loaded SOME version of Python.Runtime - // So now we get the PythonEngine and execute the InitExt method on it. - Type pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); - - return (IntPtr)pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); - } - - /// - /// Substitute for Debug.Writeline(...). Ideally we would use Debug.Writeline - /// but haven't been able to configure the TRACE from within Python. - /// - [Conditional("DEBUG")] - private static void DebugPrint(string str) - { - Console.WriteLine(str); - } -} diff --git a/src/clrmodule/Properties/AssemblyInfo.cs b/src/clrmodule/Properties/AssemblyInfo.cs deleted file mode 100644 index 5e2e05ed4..000000000 --- a/src/clrmodule/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System.Reflection; -using System.Runtime.InteropServices; - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("ae10d6a4-55c2-482f-9716-9988e6c169e3")] diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj deleted file mode 100644 index 8595fd0ba..000000000 --- a/src/clrmodule/clrmodule.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - net472 - win-x86;win-x64 - clr - - - - - - - 1.0.0 - all - runtime; build; native; contentfiles; analyzers - - - - - x86 - - - x64 - - diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 03812c6fe..7dd5a765e 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -34,11 +34,12 @@ public void Dispose() public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); - string attributeName = "abiflags"; - if (sys.HasAttr(attributeName) && !string.IsNullOrEmpty(sys.GetAttr(attributeName).ToString())) + // We can safely ignore the "m" abi flag + var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; - Assert.NotNull(Type.GetType(typeName), $"{typeName} does not exist and sys.{attributeName} is not empty"); + Assert.NotNull(Type.GetType(typeName), $"{typeName} does not exist and sys.abiflags={abiflags}"); } } } diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c deleted file mode 100644 index cdfd89342..000000000 --- a/src/monoclr/clrmod.c +++ /dev/null @@ -1,215 +0,0 @@ -// #define Py_LIMITED_API 0x03050000 -#include - -#include "stdlib.h" - -#define MONO_VERSION "v4.0.30319.1" -#define MONO_DOMAIN "Python" - -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include "dirent.h" -#include "dlfcn.h" -#include "libgen.h" -#include "alloca.h" -#endif - -typedef struct -{ - MonoDomain *domain; - MonoAssembly *pr_assm; - MonoMethod *shutdown; - const char *pr_file; - char *error; - char *init_name; - char *shutdown_name; - PyObject *module; -} PyNet_Args; - -PyNet_Args *PyNet_Init(void); -static PyNet_Args *pn_args; - -PyMODINIT_FUNC -PyInit_clr(void) -{ - pn_args = PyNet_Init(); - if (pn_args->error) - { - return NULL; - } - - return pn_args->module; -} - -void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(PyNet_Args *user_data); - -// initialize Mono and PythonNet -PyNet_Args *PyNet_Init() -{ - PyObject *pn_module; - PyObject *pn_path; - PyNet_Args *pn_args; - pn_args = (PyNet_Args *)malloc(sizeof(PyNet_Args)); - - pn_module = PyImport_ImportModule("pythonnet"); - if (pn_module == NULL) - { - pn_args->error = "Failed to import pythonnet"; - return pn_args; - } - - pn_path = PyObject_CallMethod(pn_module, "get_assembly_path", NULL); - if (pn_path == NULL) - { - Py_DecRef(pn_module); - pn_args->error = "Failed to get assembly path"; - return pn_args; - } - - pn_args->pr_file = PyUnicode_AsUTF8(pn_path); - pn_args->error = NULL; - pn_args->shutdown = NULL; - pn_args->module = NULL; - -#ifdef __linux__ - // Force preload libmono-2.0 as global. Without this, on some systems - // symbols from libmono are not found by libmononative (which implements - // some of the System.* namespaces). Since the only happened on Linux so - // far, we hardcode the library name, load the symbols into the global - // namespace and leak the handle. - dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); -#endif - - pn_args->init_name = "Python.Runtime:InitExt()"; - pn_args->shutdown_name = "Python.Runtime:Shutdown()"; - - pn_args->domain = mono_jit_init_version(MONO_DOMAIN, MONO_VERSION); - - // XXX: Skip setting config for now, should be derived from pr_file - // mono_domain_set_config(pn_args->domain, ".", "Python.Runtime.dll.config"); - - /* - * Load the default Mono configuration file, this is needed - * if you are planning on using the dllmaps defined on the - * system configuration - */ - mono_config_parse(NULL); - - main_thread_handler(pn_args); - - if (pn_args->error != NULL) - { - PyErr_SetString(PyExc_ImportError, pn_args->error); - } - return pn_args; -} - -char *PyNet_ExceptionToString(MonoObject *e); - -// Shuts down PythonNet and cleans up Mono -void PyNet_Finalize(PyNet_Args *pn_args) -{ - MonoObject *exception = NULL; - - if (pn_args->shutdown) - { - mono_runtime_invoke(pn_args->shutdown, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - } - pn_args->shutdown = NULL; - } - - if (pn_args->domain) - { - mono_jit_cleanup(pn_args->domain); - pn_args->domain = NULL; - } - free(pn_args); -} - -MonoMethod *getMethodFromClass(MonoClass *cls, char *name) -{ - MonoMethodDesc *mdesc; - MonoMethod *method; - - mdesc = mono_method_desc_new(name, 1); - method = mono_method_desc_search_in_class(mdesc, cls); - mono_method_desc_free(mdesc); - - return method; -} - -void main_thread_handler(PyNet_Args *user_data) -{ - PyNet_Args *pn_args = user_data; - MonoMethod *init; - MonoImage *pr_image; - MonoClass *pythonengine; - MonoObject *exception = NULL; - MonoObject *init_result; - - pn_args->pr_assm = mono_domain_assembly_open(pn_args->domain, pn_args->pr_file); - if (!pn_args->pr_assm) - { - pn_args->error = "Unable to load assembly"; - return; - } - - pr_image = mono_assembly_get_image(pn_args->pr_assm); - if (!pr_image) - { - pn_args->error = "Unable to get image"; - return; - } - - pythonengine = mono_class_from_name(pr_image, "Python.Runtime", "PythonEngine"); - if (!pythonengine) - { - pn_args->error = "Unable to load class PythonEngine from Python.Runtime"; - return; - } - - init = getMethodFromClass(pythonengine, pn_args->init_name); - if (!init) - { - pn_args->error = "Unable to fetch Init method from PythonEngine"; - return; - } - - pn_args->shutdown = getMethodFromClass(pythonengine, pn_args->shutdown_name); - if (!pn_args->shutdown) - { - pn_args->error = "Unable to fetch shutdown method from PythonEngine"; - return; - } - - init_result = mono_runtime_invoke(init, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - return; - } - - pn_args->module = *(PyObject**)mono_object_unbox(init_result); -} - -// Get string from a Mono exception -char *PyNet_ExceptionToString(MonoObject *e) -{ - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); - MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); - mono_method_desc_free(mdesc); - - mmethod = mono_object_get_virtual_method(e, mmethod); - MonoString *monoString = (MonoString*) mono_runtime_invoke(mmethod, e, NULL, NULL); - mono_runtime_invoke(mmethod, e, NULL, NULL); - return mono_string_to_utf8(monoString); -} diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index ef530d69a..0311dbf9a 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -30,7 +30,8 @@ - $(DefineConstants);$(ConfiguredConstants) + ..\..\pythonnet\runtime + false diff --git a/src/runtime/loader.cs b/src/runtime/loader.cs new file mode 100644 index 000000000..d5f31b247 --- /dev/null +++ b/src/runtime/loader.cs @@ -0,0 +1,83 @@ +using System.Diagnostics; +using System; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace Python.Runtime +{ + using static Runtime; + + [Obsolete("Only to be used from within Python")] + static class Loader + { + public unsafe static int Initialize(IntPtr data, int size) + { + IntPtr gs = IntPtr.Zero; + try + { + var dllPath = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + + if (!string.IsNullOrEmpty(dllPath)) + { + PythonDLL = dllPath; + } + else + { + PythonDLL = null; + } + + gs = PyGILState_Ensure(); + + // Console.WriteLine("Startup thread"); + PythonEngine.InitExt(); + // Console.WriteLine("Startup finished"); + } + catch (Exception exc) + { + Console.Error.Write( + $"Failed to initialize pythonnet: {exc}\n{exc.StackTrace}" + ); + return 1; + } + finally + { + if (gs != IntPtr.Zero) + { + PyGILState_Release(gs); + } + } + return 0; + } + + public unsafe static int Shutdown(IntPtr data, int size) + { + IntPtr gs = IntPtr.Zero; + try + { + var command = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + + if (command == "full_shutdown") + { + gs = PyGILState_Ensure(); + PythonEngine.Shutdown(); + } + } + catch (Exception exc) + { + Console.Error.Write( + $"Failed to shutdown pythonnet: {exc}\n{exc.StackTrace}" + ); + return 1; + } + finally + { + if (gs != IntPtr.Zero) + { + PyGILState_Release(gs); + } + } + return 0; + } + } +} diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 3fdd99b9a..41167e322 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -534,11 +534,5 @@ public static string[] ListAssemblies(bool verbose) } return names; } - - [ModuleFunction] - public static int _AtExit() - { - return Runtime.AtExit(); - } } } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 4c1bcefa0..9f5ed671b 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -147,7 +147,6 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) { "__instancecheck__", "__subclasscheck__", - "_AtExit", "AddReference", "FinalizeObject", "FindAssembly", diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index b5334fabc..35ea3f6d2 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -198,18 +198,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Py.SetArgv(args); } - if (mode == ShutdownMode.Normal) - { - // TOOD: Check if this can be remove completely or not. - // register the atexit callback (this doesn't use Py_AtExit as the C atexit - // callbacks are called after python is fully finalized but the python ones - // are called while the python engine is still running). - //string code = - // "import atexit, clr\n" + - // "atexit.register(clr._AtExit)\n"; - //PythonEngine.Exec(code); - } - // Load the clr.py resource into the clr module NewReference clr = Python.Runtime.ImportHook.GetCLRModule(); BorrowedReference clr_dict = Runtime.PyModule_GetDict(clr); @@ -266,7 +254,7 @@ public static IntPtr InitExt() { try { - Initialize(setSysArgv: false); + Initialize(setSysArgv: false, mode: ShutdownMode.Extension); // Trickery - when the import hook is installed into an already // running Python, the standard import machinery is still in diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 990ac2a9f..ec7f5e446 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -133,7 +133,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // If we're coming back from a domain reload or a soft shutdown, // we have previously released the thread state. Restore the main // thread state here. - PyGILState_Ensure(); + if (mode != ShutdownMode.Extension) + { + PyGILState_Ensure(); + } } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; @@ -356,7 +359,7 @@ internal static void Shutdown(ShutdownMode mode) Finalizer.Shutdown(); InternString.Shutdown(); - if (mode != ShutdownMode.Normal) + if (mode != ShutdownMode.Normal && mode != ShutdownMode.Extension) { PyGC_Collect(); if (mode == ShutdownMode.Soft) @@ -387,7 +390,10 @@ internal static void Shutdown(ShutdownMode mode) else { ResetPyMembers(); - Py_Finalize(); + if (mode != ShutdownMode.Extension) + { + Py_Finalize(); + } } } @@ -412,16 +418,6 @@ internal static ShutdownMode GetDefaultShutdownMode() return ShutdownMode.Normal; } - // called *without* the GIL acquired by clr._AtExit - internal static int AtExit() - { - lock (IsFinalizingLock) - { - IsFinalizing = true; - } - return 0; - } - private static void RunExitFuncs() { PyObject atexit; @@ -781,7 +777,7 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - + internal static void Py_IncRef(IntPtr ob) => Delegates.Py_IncRef(ob); /// @@ -789,59 +785,59 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - + internal static void Py_DecRef(IntPtr ob) => Delegates.Py_DecRef(ob); - + internal static void Py_Initialize() => Delegates.Py_Initialize(); - + internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); - + internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); - + internal static void Py_Finalize() => Delegates.Py_Finalize(); - + internal static IntPtr Py_NewInterpreter() => Delegates.Py_NewInterpreter(); - + internal static void Py_EndInterpreter(IntPtr threadState) => Delegates.Py_EndInterpreter(threadState); - + internal static IntPtr PyThreadState_New(IntPtr istate) => Delegates.PyThreadState_New(istate); - + internal static IntPtr PyThreadState_Get() => Delegates.PyThreadState_Get(); - + internal static IntPtr _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); - + internal static IntPtr PyThread_get_key_value(IntPtr key) => Delegates.PyThread_get_key_value(key); - + internal static int PyThread_get_thread_ident() => Delegates.PyThread_get_thread_ident(); - + internal static int PyThread_set_key_value(IntPtr key, IntPtr value) => Delegates.PyThread_set_key_value(key, value); - + internal static IntPtr PyThreadState_Swap(IntPtr key) => Delegates.PyThreadState_Swap(key); - + internal static IntPtr PyGILState_Ensure() => Delegates.PyGILState_Ensure(); - + internal static void PyGILState_Release(IntPtr gs) => Delegates.PyGILState_Release(gs); - + internal static IntPtr PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - + public static int Py_Main(int argc, string[] argv) { var marshaler = StrArrayMarshaler.GetInstance(null); @@ -858,67 +854,67 @@ public static int Py_Main(int argc, string[] argv) internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); - + internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); - + internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); - + internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); - + internal static void PyEval_AcquireThread(IntPtr tstate) => Delegates.PyEval_AcquireThread(tstate); - + internal static void PyEval_ReleaseThread(IntPtr tstate) => Delegates.PyEval_ReleaseThread(tstate); - + internal static IntPtr PyEval_SaveThread() => Delegates.PyEval_SaveThread(); - + internal static void PyEval_RestoreThread(IntPtr tstate) => Delegates.PyEval_RestoreThread(tstate); - + internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); - + internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); - + internal static IntPtr PyEval_GetLocals() => Delegates.PyEval_GetLocals(); - + internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); - + internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); - + internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); - + internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); - + internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); - + internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); - + internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); - + internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); - + internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); - + internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); - + internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; @@ -956,10 +952,10 @@ internal static IntPtr PyImport_ExecCodeModule(string name, IntPtr code) internal static IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod) => Delegates.PyCFunction_NewEx(ml, self, mod); - + internal static IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw) => Delegates.PyCFunction_Call(func, args, kw); - + internal static IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls) => Delegates.PyMethod_New(func, self, cls); @@ -1021,7 +1017,7 @@ internal static bool PyObject_IsIterable(IntPtr pointer) return tp_iter != IntPtr.Zero; } - + internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) { using var namePtr = new StrPtr(name, Encoding.UTF8); @@ -1034,7 +1030,7 @@ internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) return Delegates.PyObject_GetAttrString(pointer, namePtr); } - + internal static IntPtr PyObject_GetAttrString(IntPtr pointer, StrPtr name) => Delegates.PyObject_GetAttrString(pointer, name); @@ -1057,25 +1053,25 @@ internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) internal static int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value) => Delegates.PyObject_SetAttr(pointer, name, value); - + internal static IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_GetItem(pointer, key); - + internal static int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value) => Delegates.PyObject_SetItem(pointer, key, value); - + internal static int PyObject_DelItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_DelItem(pointer, key); - + internal static IntPtr PyObject_GetIter(IntPtr op) => Delegates.PyObject_GetIter(op); - + internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); - + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) => Delegates.PyObject_CallObject(pointer, args); - + internal static int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); internal static int PyObject_Compare(IntPtr value1, IntPtr value2) @@ -1103,20 +1099,20 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) return -1; } - + internal static int PyObject_IsInstance(IntPtr ob, IntPtr type) => Delegates.PyObject_IsInstance(ob, type); - + internal static int PyObject_IsSubclass(IntPtr ob, IntPtr type) => Delegates.PyObject_IsSubclass(ob, type); - + internal static int PyCallable_Check(IntPtr pointer) => Delegates.PyCallable_Check(pointer); internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); - + internal static int PyObject_Not(IntPtr pointer) => Delegates.PyObject_Not(pointer); internal static long PyObject_Size(IntPtr pointer) @@ -1124,22 +1120,22 @@ internal static long PyObject_Size(IntPtr pointer) return (long)_PyObject_Size(pointer); } - + private static IntPtr _PyObject_Size(IntPtr pointer) => Delegates._PyObject_Size(pointer); - + internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); - + internal static IntPtr PyObject_Repr(IntPtr pointer) => Delegates.PyObject_Repr(pointer); - + internal static IntPtr PyObject_Str(IntPtr pointer) => Delegates.PyObject_Str(pointer); - + internal static IntPtr PyObject_Unicode(IntPtr pointer) => Delegates.PyObject_Unicode(pointer); - + internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); #if PYTHON_WITH_PYDEBUG @@ -1151,13 +1147,13 @@ internal static long PyObject_Size(IntPtr pointer) // Python buffer API //==================================================================== - + internal static int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, ref view, flags); - + internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); - + internal static IntPtr PyBuffer_SizeFromFormat(string format) { using var formatPtr = new StrPtr(format, Encoding.ASCII); @@ -1166,35 +1162,35 @@ internal static IntPtr PyBuffer_SizeFromFormat(string format) internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); - + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); - + internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); - + internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); - + internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); - + internal static int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); //==================================================================== // Python number API //==================================================================== - + internal static IntPtr PyNumber_Int(IntPtr ob) => Delegates.PyNumber_Int(ob); - + internal static IntPtr PyNumber_Long(IntPtr ob) => Delegates.PyNumber_Long(ob); - + internal static IntPtr PyNumber_Float(IntPtr ob) => Delegates.PyNumber_Float(ob); - + internal static bool PyNumber_Check(IntPtr ob) => Delegates.PyNumber_Check(ob); internal static bool PyInt_Check(BorrowedReference ob) @@ -1221,25 +1217,25 @@ internal static IntPtr PyInt_FromInt64(long value) return PyInt_FromLong(v); } - + private static IntPtr PyInt_FromLong(IntPtr value) => Delegates.PyInt_FromLong(value); - + internal static int PyInt_AsLong(IntPtr value) => Delegates.PyInt_AsLong(value); - + internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; } - + internal static IntPtr PyLong_FromLong(long value) => Delegates.PyLong_FromLong(value); - + internal static IntPtr PyLong_FromUnsignedLong32(uint value) => Delegates.PyLong_FromUnsignedLong32(value); - + internal static IntPtr PyLong_FromUnsignedLong64(ulong value) => Delegates.PyLong_FromUnsignedLong64(value); internal static IntPtr PyLong_FromUnsignedLong(object value) @@ -1250,16 +1246,16 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); } - + internal static IntPtr PyLong_FromDouble(double value) => Delegates.PyLong_FromDouble(value); - + internal static IntPtr PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); - + internal static IntPtr PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); - + internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) { using var valPtr = new StrPtr(value, Encoding.UTF8); @@ -1267,11 +1263,11 @@ internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) } - + internal static nuint PyLong_AsUnsignedSize_t(IntPtr value) => Delegates.PyLong_AsUnsignedSize_t(value); - + internal static nint PyLong_AsSignedSize_t(IntPtr value) => Delegates.PyLong_AsSignedSize_t(new BorrowedReference(value)); - + internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); /// @@ -1282,7 +1278,7 @@ internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) /// In most cases you need to check that value is an instance of PyLongObject /// before using this function using . /// - + internal static long PyExplicitlyConvertToInt64(IntPtr value) => Delegates.PyExplicitlyConvertToInt64(value); internal static ulong PyLong_AsUnsignedLongLong(IntPtr value) => Delegates.PyLong_AsUnsignedLongLong(value); @@ -1301,91 +1297,91 @@ internal static bool PyFloat_Check(IntPtr ob) /// /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). /// - + internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); - + internal static IntPtr PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); - + internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); - + internal static double PyFloat_AsDouble(IntPtr ob) => Delegates.PyFloat_AsDouble(ob); - + internal static IntPtr PyNumber_Add(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Add(o1, o2); - + internal static IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Subtract(o1, o2); - + internal static IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Multiply(o1, o2); - + internal static IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_TrueDivide(o1, o2); - + internal static IntPtr PyNumber_And(IntPtr o1, IntPtr o2) => Delegates.PyNumber_And(o1, o2); - + internal static IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Xor(o1, o2); - + internal static IntPtr PyNumber_Or(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Or(o1, o2); - + internal static IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Lshift(o1, o2); - + internal static IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Rshift(o1, o2); - + internal static IntPtr PyNumber_Power(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Power(o1, o2); - + internal static IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Remainder(o1, o2); - + internal static IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); - + internal static IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); - + internal static IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); - + internal static IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); - + internal static IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); - + internal static IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceXor(o1, o2); - + internal static IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceOr(o1, o2); - + internal static IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); - + internal static IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); - + internal static IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlacePower(o1, o2); - + internal static IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); - + internal static IntPtr PyNumber_Negative(IntPtr o1) => Delegates.PyNumber_Negative(o1); - + internal static IntPtr PyNumber_Positive(IntPtr o1) => Delegates.PyNumber_Positive(o1); - + internal static IntPtr PyNumber_Invert(IntPtr o1) => Delegates.PyNumber_Invert(o1); @@ -1393,9 +1389,9 @@ internal static bool PyFloat_Check(IntPtr ob) // Python sequence API //==================================================================== - + internal static bool PySequence_Check(IntPtr pointer) => Delegates.PySequence_Check(pointer); - + internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) @@ -1403,7 +1399,7 @@ internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) return PySequence_SetItem(pointer, new IntPtr(index), value); } - + private static int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PySequence_SetItem(pointer, index, value); internal static int PySequence_DelItem(IntPtr pointer, long index) @@ -1411,7 +1407,7 @@ internal static int PySequence_DelItem(IntPtr pointer, long index) return PySequence_DelItem(pointer, new IntPtr(index)); } - + private static int PySequence_DelItem(IntPtr pointer, IntPtr index) => Delegates.PySequence_DelItem(pointer, index); internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) @@ -1419,7 +1415,7 @@ internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - + private static IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) @@ -1427,7 +1423,7 @@ internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); } - + private static int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) @@ -1435,17 +1431,17 @@ internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - + private static int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); [Obsolete] internal static nint PySequence_Size(IntPtr pointer) => PySequence_Size(new BorrowedReference(pointer)); internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); - + internal static int PySequence_Contains(IntPtr pointer, IntPtr item) => Delegates.PySequence_Contains(pointer, item); - + internal static IntPtr PySequence_Concat(IntPtr pointer, IntPtr other) => Delegates.PySequence_Concat(pointer, other); internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) @@ -1453,10 +1449,10 @@ internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) return PySequence_Repeat(pointer, new IntPtr(count)); } - + private static IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count) => Delegates.PySequence_Repeat(pointer, count); - + internal static int PySequence_Index(IntPtr pointer, IntPtr item) => Delegates.PySequence_Index(pointer, item); internal static long PySequence_Count(IntPtr pointer, IntPtr value) @@ -1464,13 +1460,13 @@ internal static long PySequence_Count(IntPtr pointer, IntPtr value) return (long)_PySequence_Count(pointer, value); } - + private static IntPtr _PySequence_Count(IntPtr pointer, IntPtr value) => Delegates._PySequence_Count(pointer, value); - + internal static IntPtr PySequence_Tuple(IntPtr pointer) => Delegates.PySequence_Tuple(pointer); - + internal static IntPtr PySequence_List(IntPtr pointer) => Delegates.PySequence_List(pointer); @@ -1500,7 +1496,7 @@ internal static IntPtr PyString_FromString(string value) return PyUnicode_FromKindAndData(2, (IntPtr)ptr, value.Length); } - + internal static IntPtr EmptyPyBytes() { byte* bytes = stackalloc byte[1]; @@ -1513,7 +1509,7 @@ internal static long PyBytes_Size(IntPtr op) return (long)_PyBytes_Size(op); } - + private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) @@ -1527,10 +1523,10 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) return PyUnicode_FromStringAndSize(value, new IntPtr(size)); } - + private static IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size) => Delegates.PyUnicode_FromStringAndSize(value, size); - + internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode); internal static bool PyUnicode_Check(IntPtr ob) @@ -1538,10 +1534,10 @@ internal static bool PyUnicode_Check(IntPtr ob) return PyObject_TYPE(ob) == PyUnicodeType; } - + internal static IntPtr PyUnicode_FromObject(IntPtr ob) => Delegates.PyUnicode_FromObject(ob); - + internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size) @@ -1549,7 +1545,7 @@ internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size) return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); } - + private static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, IntPtr size) => Delegates.PyUnicode_FromKindAndData(kind, s, size); @@ -1559,7 +1555,7 @@ internal static IntPtr PyUnicode_FromUnicode(string s, long size) return PyUnicode_FromKindAndData(2, (IntPtr)ptr, size); } - + internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax(); internal static long PyUnicode_GetSize(IntPtr ob) @@ -1567,10 +1563,10 @@ internal static long PyUnicode_GetSize(IntPtr ob) return (long)_PyUnicode_GetSize(ob); } - + private static IntPtr _PyUnicode_GetSize(IntPtr ob) => Delegates._PyUnicode_GetSize(ob); - + internal static IntPtr PyUnicode_AsUnicode(IntPtr ob) => Delegates.PyUnicode_AsUnicode(ob); internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); @@ -1583,7 +1579,7 @@ internal static IntPtr PyUnicode_FromString(string s) return PyUnicode_FromUnicode(s, s.Length); } - + internal static IntPtr PyUnicode_InternFromString(string s) { using var ptr = new StrPtr(s, Encoding.UTF8); @@ -1634,13 +1630,13 @@ internal static bool PyDict_Check(IntPtr ob) return PyObject_TYPE(ob) == PyDictType; } - + internal static IntPtr PyDict_New() => Delegates.PyDict_New(); - + internal static int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue) => Delegates.PyDict_Next(p, out ppos, out pkey, out pvalue); - + internal static IntPtr PyDictProxy_New(IntPtr dict) => Delegates.PyDictProxy_New(dict); /// @@ -1694,7 +1690,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); - + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) { using var keyPtr = new StrPtr(key, Encoding.UTF8); @@ -1710,19 +1706,19 @@ internal static IntPtr PyDict_Keys(IntPtr pointer) .DangerousMoveToPointerOrNull(); internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); - + internal static IntPtr PyDict_Values(IntPtr pointer) => Delegates.PyDict_Values(pointer); - + internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); - + internal static IntPtr PyDict_Copy(IntPtr pointer) => Delegates.PyDict_Copy(pointer); - + internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); - + internal static void PyDict_Clear(IntPtr pointer) => Delegates.PyDict_Clear(pointer); internal static long PyDict_Size(IntPtr pointer) @@ -1730,19 +1726,19 @@ internal static long PyDict_Size(IntPtr pointer) return (long)_PyDict_Size(pointer); } - + internal static IntPtr _PyDict_Size(IntPtr pointer) => Delegates._PyDict_Size(pointer); internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); - + internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); /// /// Return 1 if found, 0 if not found, and -1 if an error is encountered. /// - + internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); //==================================================================== @@ -1759,10 +1755,10 @@ internal static IntPtr PyList_New(long size) return PyList_New(new IntPtr(size)); } - + private static IntPtr PyList_New(IntPtr size) => Delegates.PyList_New(size); - + internal static IntPtr PyList_AsTuple(IntPtr pointer) => Delegates.PyList_AsTuple(pointer); internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, long index) @@ -1770,7 +1766,7 @@ internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, long return PyList_GetItem(pointer, new IntPtr(index)); } - + private static BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyList_GetItem(pointer, index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) @@ -1778,7 +1774,7 @@ internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) return PyList_SetItem(pointer, new IntPtr(index), value); } - + private static int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyList_SetItem(pointer, index, value); internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) @@ -1786,16 +1782,16 @@ internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr return PyList_Insert(pointer, new IntPtr(index), value); } - + private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); - + internal static int PyList_Append(BorrowedReference pointer, IntPtr value) => Delegates.PyList_Append(pointer, value); - + internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); - + internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) @@ -1803,7 +1799,7 @@ internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - + private static IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyList_GetSlice(pointer, start, end); internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) @@ -1811,10 +1807,10 @@ internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); } - + private static int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value) => Delegates.PyList_SetSlice(pointer, start, end, value); - + internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); //==================================================================== @@ -1835,7 +1831,7 @@ internal static IntPtr PyTuple_New(long size) return PyTuple_New(new IntPtr(size)); } - + private static IntPtr PyTuple_New(IntPtr size) => Delegates.PyTuple_New(size); internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, long index) @@ -1846,7 +1842,7 @@ internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) .DangerousGetAddressOrNull(); } - + private static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyTuple_GetItem(pointer, index); internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) @@ -1854,7 +1850,7 @@ internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) return PyTuple_SetItem(pointer, new IntPtr(index), value); } - + private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) @@ -1862,7 +1858,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - + private static IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyTuple_GetSlice(pointer, start, end); @@ -1881,7 +1877,7 @@ internal static bool PyIter_Check(IntPtr pointer) return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } - + internal static IntPtr PyIter_Next(IntPtr pointer) => Delegates.PyIter_Next(new BorrowedReference(pointer)).DangerousMoveToPointerOrNull(); internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); @@ -1891,7 +1887,7 @@ internal static IntPtr PyIter_Next(IntPtr pointer) // Python module API //==================================================================== - + internal static NewReference PyModule_New(string name) { using var namePtr = new StrPtr(name, Encoding.UTF8); @@ -1903,18 +1899,18 @@ internal static string PyModule_GetName(IntPtr module) internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); - + internal static string PyModule_GetFilename(IntPtr module) => Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8); #if PYTHON_WITH_PYDEBUG [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] #else - + #endif internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver); - + internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); /// @@ -1929,7 +1925,7 @@ internal static IntPtr PyImport_ImportModule(string name) internal static IntPtr PyImport_ReloadModule(IntPtr module) => Delegates.PyImport_ReloadModule(module); - + internal static BorrowedReference PyImport_AddModule(string name) { using var namePtr = new StrPtr(name, Encoding.UTF8); @@ -1938,7 +1934,7 @@ internal static BorrowedReference PyImport_AddModule(string name) internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); - + internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) { var marshaler = StrArrayMarshaler.GetInstance(null); @@ -1979,7 +1975,7 @@ internal static bool PyType_Check(IntPtr ob) return PyObject_TypeCheck(ob, PyTypeType); } - + internal static void PyType_Modified(IntPtr type) => Delegates.PyType_Modified(type); internal static bool PyType_IsSubtype(BorrowedReference t1, IntPtr ofType) => PyType_IsSubtype(t1, new BorrowedReference(ofType)); @@ -2000,7 +1996,7 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe return (type == ofType) || PyType_IsSubtype(type, ofType); } - + internal static IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw) => Delegates.PyType_GenericNew(type, args, kw); internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) @@ -2008,37 +2004,37 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) return PyType_GenericAlloc(type, new IntPtr(n)); } - + private static IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n) => Delegates.PyType_GenericAlloc(type, n); /// /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// - + internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type); - + internal static IntPtr _PyType_Lookup(IntPtr type, IntPtr name) => Delegates._PyType_Lookup(type, name); - + internal static IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name) => Delegates.PyObject_GenericGetAttr(obj, name); - + internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - + internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - + internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); - + internal static void PyObject_GC_Track(IntPtr tp) => Delegates.PyObject_GC_Track(tp); - + internal static void PyObject_GC_UnTrack(IntPtr tp) => Delegates.PyObject_GC_UnTrack(tp); - + internal static void _PyObject_Dump(IntPtr ob) => Delegates._PyObject_Dump(ob); //==================================================================== @@ -2050,7 +2046,7 @@ internal static IntPtr PyMem_Malloc(long size) return PyMem_Malloc(new IntPtr(size)); } - + private static IntPtr PyMem_Malloc(IntPtr size) => Delegates.PyMem_Malloc(size); internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) @@ -2058,10 +2054,10 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) return PyMem_Realloc(ptr, new IntPtr(size)); } - + private static IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size) => Delegates.PyMem_Realloc(ptr, size); - + internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); @@ -2069,7 +2065,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) // Python exception API //==================================================================== - + internal static void PyErr_SetString(IntPtr ob, string message) { using var msgPtr = new StrPtr(message, Encoding.UTF8); @@ -2078,40 +2074,40 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); - + internal static IntPtr PyErr_SetFromErrno(IntPtr ob) => Delegates.PyErr_SetFromErrno(ob); - + internal static void PyErr_SetNone(IntPtr ob) => Delegates.PyErr_SetNone(ob); - + internal static int PyErr_ExceptionMatches(IntPtr exception) => Delegates.PyErr_ExceptionMatches(exception); - + internal static int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val) => Delegates.PyErr_GivenExceptionMatches(ob, val); - + internal static void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb) => Delegates.PyErr_NormalizeException(ref ob, ref val, ref tb); - + internal static IntPtr PyErr_Occurred() => Delegates.PyErr_Occurred(); - + internal static void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb) => Delegates.PyErr_Fetch(out ob, out val, out tb); - + internal static void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb) => Delegates.PyErr_Restore(ob, val, tb); - + internal static void PyErr_Clear() => Delegates.PyErr_Clear(); - + internal static void PyErr_Print() => Delegates.PyErr_Print(); /// /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. /// - + internal static void PyException_SetCause(IntPtr ex, IntPtr cause) => Delegates.PyException_SetCause(ex, cause); //==================================================================== @@ -2121,7 +2117,7 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); - + internal static int PyCell_Set(BorrowedReference cell, IntPtr value) => Delegates.PyCell_Set(cell, value); //==================================================================== @@ -2134,7 +2130,7 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; - + internal static IntPtr PyGC_Collect() => Delegates.PyGC_Collect(); internal static IntPtr _Py_AS_GC(BorrowedReference ob) @@ -2196,18 +2192,18 @@ internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr na // Miscellaneous //==================================================================== - + internal static IntPtr PyMethod_Self(IntPtr ob) => Delegates.PyMethod_Self(ob); - + internal static IntPtr PyMethod_Function(IntPtr ob) => Delegates.PyMethod_Function(ob); - + internal static int Py_AddPendingCall(IntPtr func, IntPtr arg) => Delegates.Py_AddPendingCall(func, arg); - + internal static int PyThreadState_SetAsyncExcLLP64(uint id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); - + internal static int PyThreadState_SetAsyncExcLP64(ulong id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); @@ -2789,6 +2785,7 @@ public enum ShutdownMode Normal, Soft, Reload, + Extension, } diff --git a/src/tests/test_method.py b/src/tests/test_method.py index 2826ad467..9bdb571c0 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -899,9 +899,6 @@ def test_object_in_multiparam_exception(): c = e.__cause__ assert c.GetType().FullName == 'System.AggregateException' assert len(c.InnerExceptions) == 2 - message = 'One or more errors occurred.' - s = str(c) - assert s[0:len(message)] == message def test_case_sensitive(): """Test that case-sensitivity is respected. GH#81""" diff --git a/src/tests/test_sysargv.py b/src/tests/test_sysargv.py index dd62bc632..d856ec902 100644 --- a/src/tests/test_sysargv.py +++ b/src/tests/test_sysargv.py @@ -2,6 +2,7 @@ import sys from subprocess import check_output +from ast import literal_eval def test_sys_argv_state(filepath): @@ -11,5 +12,5 @@ def test_sys_argv_state(filepath): script = filepath("argv-fixture.py") out = check_output([sys.executable, script, "foo", "bar"]) - assert b"foo" in out - assert b"bar" in out + out = literal_eval(out.decode("ascii")) + assert out[-2:] == ["foo", "bar"] diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 0d5b83b30..0c80c1904 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -225,8 +225,6 @@ def preprocess_python_headers(): if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) - if "m" in sys.abiflags: - defines.extend(("-D", "PYTHON_WITH_PYMALLOC")) if "u" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) @@ -245,7 +243,7 @@ def preprocess_python_headers(): def gen_interop_head(writer): filename = os.path.basename(__file__) - abi_flags = getattr(sys, "abiflags", "") + abi_flags = getattr(sys, "abiflags", "").replace("m", "") py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) class_definition = """ // Auto-generated by %s. From fdb71447a4374f54dc0c346474b90c55e5796e25 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 12:40:54 +0100 Subject: [PATCH 046/151] Add Changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1fd340e..9bee653e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ when .NET expects an integer [#1342][i1342] - BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. - Sign Runtime DLL with a strong name +- Implement loading through `clr_loader` instead of the included `ClrModule`, enables + support for .NET Core ### Fixed From f01a78c6fd7bb34c2461e803e6ee1d757e17740b Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 13:11:06 +0100 Subject: [PATCH 047/151] Fix domain reload tests and activate them on macOS --- src/domain_tests/test_domain_reload.py | 31 +++++++++----------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index fa6f42b9e..a7cd2fa4d 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -4,6 +4,12 @@ import pytest +from pythonnet.find_libpython import find_libpython +libpython = find_libpython() + +pytestmark = pytest.mark.xfail(libpython is None, reason="Can't find suitable libpython") + + def _run_test(testname): dirname = os.path.split(__file__)[0] exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') @@ -12,90 +18,73 @@ def _run_test(testname): if platform.system() != 'Windows': args = ['mono'] + args - proc = subprocess.Popen(args) + env = os.environ.copy() + env["PYTHONNET_PYDLL"] = libpython + + proc = subprocess.Popen(args, env=env) proc.wait() assert proc.returncode == 0 -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class(): _run_test('class_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_static_function(): _run_test('static_member_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_function(): _run_test('member_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_field(): _run_test('field_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_property(): _run_test('property_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_namespace(): _run_test('namespace_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_visibility_change(): _run_test("field_visibility_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_visibility_change(): _run_test("method_visibility_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_property_visibility_change(): _run_test("property_visibility_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_class_visibility_change(): _run_test("class_visibility_change") @pytest.mark.skip(reason='FIXME: Domain reload fails when Python points to a .NET object which points back to Python objects') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_parameters_change(): _run_test("method_parameters_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_return_type_change(): _run_test("method_return_type_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_type_change(): _run_test("field_type_change") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="Events not yet serializable") def test_rename_event(): _run_test('event_rename') -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): _run_test("construct_removed_class") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_out_to_ref_param(): _run_test("out_to_ref_param") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_ref_to_out_param(): _run_test("ref_to_out_param") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_ref_to_in_param(): _run_test("ref_to_in_param") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_in_to_ref_param(): _run_test("in_to_ref_param") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_nested_type(): _run_test("nested_type") From 0d7e43aedf07c7ac897c921cbdec00af15367b37 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 13 Feb 2021 23:07:28 +0100 Subject: [PATCH 048/151] Run tests on .NET Core --- .github/workflows/main.yml | 14 ++- pyproject.toml | 6 ++ setup.cfg | 4 - src/testing/Python.Test.csproj | 7 +- src/tests/conftest.py | 56 ---------- src/tests/fixtures/netstandard2.0/.gitkeep | 0 {src/tests => tests}/__init__.py | 0 {src/tests => tests}/_missing_import.py | 0 tests/conftest.py | 101 ++++++++++++++++++ {src/tests => tests}/fixtures/.gitkeep | 0 {src/tests => tests}/fixtures/argv-fixture.py | 0 {src/tests => tests}/importtest.py | 0 {src/tests => tests}/leaktest.py | 0 {src/tests => tests}/profile.py | 0 {src/tests => tests}/runtests.py | 0 {src/tests => tests}/stress.py | 0 {src/tests => tests}/stresstest.py | 0 {src/tests => tests}/test_array.py | 0 {src/tests => tests}/test_callback.py | 0 {src/tests => tests}/test_class.py | 0 {src/tests => tests}/test_clrmethod.py | 0 {src/tests => tests}/test_constructors.py | 0 {src/tests => tests}/test_conversion.py | 0 {src/tests => tests}/test_delegate.py | 0 {src/tests => tests}/test_docstring.py | 0 {src/tests => tests}/test_engine.py | 0 {src/tests => tests}/test_enum.py | 0 {src/tests => tests}/test_event.py | 0 {src/tests => tests}/test_exceptions.py | 0 {src/tests => tests}/test_field.py | 0 {src/tests => tests}/test_generic.py | 0 {src/tests => tests}/test_import.py | 0 {src/tests => tests}/test_indexer.py | 0 {src/tests => tests}/test_interface.py | 0 {src/tests => tests}/test_method.py | 0 {src/tests => tests}/test_module.py | 0 {src/tests => tests}/test_mp_length.py | 0 {src/tests => tests}/test_property.py | 0 {src/tests => tests}/test_recursive_types.py | 0 {src/tests => tests}/test_repr.py | 0 {src/tests => tests}/test_subclass.py | 0 {src/tests => tests}/test_sysargv.py | 0 {src/tests => tests}/test_thread.py | 0 {src/tests => tests}/tests.pyproj | 0 {src/tests => tests}/utils.py | 0 45 files changed, 120 insertions(+), 68 deletions(-) delete mode 100644 setup.cfg delete mode 100644 src/tests/conftest.py delete mode 100644 src/tests/fixtures/netstandard2.0/.gitkeep rename {src/tests => tests}/__init__.py (100%) rename {src/tests => tests}/_missing_import.py (100%) create mode 100644 tests/conftest.py rename {src/tests => tests}/fixtures/.gitkeep (100%) rename {src/tests => tests}/fixtures/argv-fixture.py (100%) rename {src/tests => tests}/importtest.py (100%) rename {src/tests => tests}/leaktest.py (100%) rename {src/tests => tests}/profile.py (100%) rename {src/tests => tests}/runtests.py (100%) rename {src/tests => tests}/stress.py (100%) rename {src/tests => tests}/stresstest.py (100%) rename {src/tests => tests}/test_array.py (100%) rename {src/tests => tests}/test_callback.py (100%) rename {src/tests => tests}/test_class.py (100%) rename {src/tests => tests}/test_clrmethod.py (100%) rename {src/tests => tests}/test_constructors.py (100%) rename {src/tests => tests}/test_conversion.py (100%) rename {src/tests => tests}/test_delegate.py (100%) rename {src/tests => tests}/test_docstring.py (100%) rename {src/tests => tests}/test_engine.py (100%) rename {src/tests => tests}/test_enum.py (100%) rename {src/tests => tests}/test_event.py (100%) rename {src/tests => tests}/test_exceptions.py (100%) rename {src/tests => tests}/test_field.py (100%) rename {src/tests => tests}/test_generic.py (100%) rename {src/tests => tests}/test_import.py (100%) rename {src/tests => tests}/test_indexer.py (100%) rename {src/tests => tests}/test_interface.py (100%) rename {src/tests => tests}/test_method.py (100%) rename {src/tests => tests}/test_module.py (100%) rename {src/tests => tests}/test_mp_length.py (100%) rename {src/tests => tests}/test_property.py (100%) rename {src/tests => tests}/test_recursive_types.py (100%) rename {src/tests => tests}/test_repr.py (100%) rename {src/tests => tests}/test_subclass.py (100%) rename {src/tests => tests}/test_sysargv.py (100%) rename {src/tests => tests}/test_thread.py (100%) rename {src/tests => tests}/tests.pyproj (100%) rename {src/tests => tests}/utils.py (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10959ea4f..2dd75c529 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,9 +55,17 @@ jobs: if: ${{ matrix.os == 'windows' }} run: | python -m pythonnet.find_libpython --export | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Python Tests - run: pytest + + - name: Python Tests (Mono) + if: ${{ matrix.os != 'windows' }} + run: pytest --runtime mono + + - name: Python Tests (.NET Core) + run: pytest --runtime netcore + + - name: Python Tests (.NET Framework) + if: ${{ matrix.os == 'windows' }} + run: pytest --runtime netfx - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ diff --git a/pyproject.toml b/pyproject.toml index 83a58d126..9bcf734c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] requires = ["setuptools>=42", "wheel", "pycparser"] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +xfail_strict = true +testpaths = [ + "tests", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 19c6f9fc9..000000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[tool:pytest] -xfail_strict = True -# -r fsxX: show extra summary info for: (f)ailed, (s)kip, (x)failed, (X)passed -addopts = -r fsxX --color=yes --durations=5 diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index e6e11c1da..4b7e4d93b 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,14 +1,11 @@ - netstandard2.0 + netstandard2.0;net5.0 true + true - - - - diff --git a/src/tests/conftest.py b/src/tests/conftest.py deleted file mode 100644 index 17085e3ef..000000000 --- a/src/tests/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -# TODO: move tests one out of src to project root. -# TODO: travis has numpy on their workers. Maybe add tests? - -"""Helpers for testing.""" - -import ctypes -import os -import sys -import sysconfig - -import pytest - -# Add path for `Python.Test` -cwd = os.path.dirname(__file__) -fixtures_path = os.path.join(cwd, "fixtures") - -BUILD_TEST = True -if BUILD_TEST: - from subprocess import check_call - test_proj_path = os.path.join(cwd, "..", "testing") - check_call(["dotnet", "build", test_proj_path, "-o", fixtures_path]) - -sys.path.append(fixtures_path) - -import clr - -# Add References for tests -clr.AddReference("Python.Test") -clr.AddReference("System.Collections") -clr.AddReference("System.Data") - - -def pytest_report_header(config): - """Generate extra report headers""" - # FIXME: https://github.com/pytest-dev/pytest/issues/2257 - is_64bits = sys.maxsize > 2**32 - arch = "x64" if is_64bits else "x86" - ucs = ctypes.sizeof(ctypes.c_wchar) - libdir = sysconfig.get_config_var("LIBDIR") - shared = bool(sysconfig.get_config_var("Py_ENABLE_SHARED")) - - header = ("Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}, " - "Py_ENABLE_SHARED: {shared}".format(**locals())) - return header - - -@pytest.fixture() -def filepath(): - """Returns full filepath for file in `fixtures` directory.""" - - def make_filepath(filename): - # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture - return os.path.join(fixtures_path, filename) - - return make_filepath diff --git a/src/tests/fixtures/netstandard2.0/.gitkeep b/src/tests/fixtures/netstandard2.0/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/tests/__init__.py b/tests/__init__.py similarity index 100% rename from src/tests/__init__.py rename to tests/__init__.py diff --git a/src/tests/_missing_import.py b/tests/_missing_import.py similarity index 100% rename from src/tests/_missing_import.py rename to tests/_missing_import.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..aa57f2a1f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# TODO: move tests one out of src to project root. +# TODO: travis has numpy on their workers. Maybe add tests? + +"""Helpers for testing.""" + +import ctypes +import os +import sys +import sysconfig +from subprocess import check_call +from tempfile import mkdtemp +import shutil + +import pytest + +from pythonnet import set_runtime + +# Add path for `Python.Test` +cwd = os.path.dirname(__file__) +fixtures_path = os.path.join(cwd, "fixtures") +sys.path.append(fixtures_path) + +def pytest_addoption(parser): + parser.addoption( + "--runtime", + action="store", + default="default", + help="Must be one of default, netcore, netfx and mono" + ) + +def pytest_configure(config): + global bin_path + runtime_opt = config.getoption("runtime") + + test_proj_path = os.path.join(cwd, "..", "src", "testing") + + if runtime_opt not in ["netcore", "netfx", "mono", "default"]: + raise RuntimeError(f"Invalid runtime: {runtime_opt}") + + bin_path = mkdtemp() + + # tmpdir_factory.mktemp(f"pythonnet-{runtime_opt}") + + fw = "net5.0" if runtime_opt == "netcore" else "netstandard2.0" + + check_call(["dotnet", "publish", "-f", fw, "-o", bin_path, test_proj_path]) + + sys.path.append(bin_path) + + if runtime_opt == "default": + pass + elif runtime_opt == "netfx": + from clr_loader import get_netfx + runtime = get_netfx() + set_runtime(runtime) + elif runtime_opt == "mono": + from clr_loader import get_mono + runtime = get_mono() + set_runtime(runtime) + elif runtime_opt == "netcore": + from clr_loader import get_coreclr + rt_config_path = os.path.join(bin_path, "Python.Test.runtimeconfig.json") + runtime = get_coreclr(rt_config_path) + set_runtime(runtime) + + import clr + clr.AddReference("Python.Test") + clr.AddReference("System") + clr.AddReference("System.Collections") + clr.AddReference("System.Data") + clr.AddReference("System.Xml") + + +def pytest_unconfigure(config): + global bin_path + shutil.rmtree(bin_path) + +def pytest_report_header(config): + """Generate extra report headers""" + # FIXME: https://github.com/pytest-dev/pytest/issues/2257 + is_64bits = sys.maxsize > 2**32 + arch = "x64" if is_64bits else "x86" + ucs = ctypes.sizeof(ctypes.c_wchar) + libdir = sysconfig.get_config_var("LIBDIR") + shared = bool(sysconfig.get_config_var("Py_ENABLE_SHARED")) + + header = ("Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}, " + "Py_ENABLE_SHARED: {shared}".format(**locals())) + return header + + +@pytest.fixture() +def filepath(): + """Returns full filepath for file in `fixtures` directory.""" + + def make_filepath(filename): + # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture + return os.path.join(fixtures_path, filename) + + return make_filepath diff --git a/src/tests/fixtures/.gitkeep b/tests/fixtures/.gitkeep similarity index 100% rename from src/tests/fixtures/.gitkeep rename to tests/fixtures/.gitkeep diff --git a/src/tests/fixtures/argv-fixture.py b/tests/fixtures/argv-fixture.py similarity index 100% rename from src/tests/fixtures/argv-fixture.py rename to tests/fixtures/argv-fixture.py diff --git a/src/tests/importtest.py b/tests/importtest.py similarity index 100% rename from src/tests/importtest.py rename to tests/importtest.py diff --git a/src/tests/leaktest.py b/tests/leaktest.py similarity index 100% rename from src/tests/leaktest.py rename to tests/leaktest.py diff --git a/src/tests/profile.py b/tests/profile.py similarity index 100% rename from src/tests/profile.py rename to tests/profile.py diff --git a/src/tests/runtests.py b/tests/runtests.py similarity index 100% rename from src/tests/runtests.py rename to tests/runtests.py diff --git a/src/tests/stress.py b/tests/stress.py similarity index 100% rename from src/tests/stress.py rename to tests/stress.py diff --git a/src/tests/stresstest.py b/tests/stresstest.py similarity index 100% rename from src/tests/stresstest.py rename to tests/stresstest.py diff --git a/src/tests/test_array.py b/tests/test_array.py similarity index 100% rename from src/tests/test_array.py rename to tests/test_array.py diff --git a/src/tests/test_callback.py b/tests/test_callback.py similarity index 100% rename from src/tests/test_callback.py rename to tests/test_callback.py diff --git a/src/tests/test_class.py b/tests/test_class.py similarity index 100% rename from src/tests/test_class.py rename to tests/test_class.py diff --git a/src/tests/test_clrmethod.py b/tests/test_clrmethod.py similarity index 100% rename from src/tests/test_clrmethod.py rename to tests/test_clrmethod.py diff --git a/src/tests/test_constructors.py b/tests/test_constructors.py similarity index 100% rename from src/tests/test_constructors.py rename to tests/test_constructors.py diff --git a/src/tests/test_conversion.py b/tests/test_conversion.py similarity index 100% rename from src/tests/test_conversion.py rename to tests/test_conversion.py diff --git a/src/tests/test_delegate.py b/tests/test_delegate.py similarity index 100% rename from src/tests/test_delegate.py rename to tests/test_delegate.py diff --git a/src/tests/test_docstring.py b/tests/test_docstring.py similarity index 100% rename from src/tests/test_docstring.py rename to tests/test_docstring.py diff --git a/src/tests/test_engine.py b/tests/test_engine.py similarity index 100% rename from src/tests/test_engine.py rename to tests/test_engine.py diff --git a/src/tests/test_enum.py b/tests/test_enum.py similarity index 100% rename from src/tests/test_enum.py rename to tests/test_enum.py diff --git a/src/tests/test_event.py b/tests/test_event.py similarity index 100% rename from src/tests/test_event.py rename to tests/test_event.py diff --git a/src/tests/test_exceptions.py b/tests/test_exceptions.py similarity index 100% rename from src/tests/test_exceptions.py rename to tests/test_exceptions.py diff --git a/src/tests/test_field.py b/tests/test_field.py similarity index 100% rename from src/tests/test_field.py rename to tests/test_field.py diff --git a/src/tests/test_generic.py b/tests/test_generic.py similarity index 100% rename from src/tests/test_generic.py rename to tests/test_generic.py diff --git a/src/tests/test_import.py b/tests/test_import.py similarity index 100% rename from src/tests/test_import.py rename to tests/test_import.py diff --git a/src/tests/test_indexer.py b/tests/test_indexer.py similarity index 100% rename from src/tests/test_indexer.py rename to tests/test_indexer.py diff --git a/src/tests/test_interface.py b/tests/test_interface.py similarity index 100% rename from src/tests/test_interface.py rename to tests/test_interface.py diff --git a/src/tests/test_method.py b/tests/test_method.py similarity index 100% rename from src/tests/test_method.py rename to tests/test_method.py diff --git a/src/tests/test_module.py b/tests/test_module.py similarity index 100% rename from src/tests/test_module.py rename to tests/test_module.py diff --git a/src/tests/test_mp_length.py b/tests/test_mp_length.py similarity index 100% rename from src/tests/test_mp_length.py rename to tests/test_mp_length.py diff --git a/src/tests/test_property.py b/tests/test_property.py similarity index 100% rename from src/tests/test_property.py rename to tests/test_property.py diff --git a/src/tests/test_recursive_types.py b/tests/test_recursive_types.py similarity index 100% rename from src/tests/test_recursive_types.py rename to tests/test_recursive_types.py diff --git a/src/tests/test_repr.py b/tests/test_repr.py similarity index 100% rename from src/tests/test_repr.py rename to tests/test_repr.py diff --git a/src/tests/test_subclass.py b/tests/test_subclass.py similarity index 100% rename from src/tests/test_subclass.py rename to tests/test_subclass.py diff --git a/src/tests/test_sysargv.py b/tests/test_sysargv.py similarity index 100% rename from src/tests/test_sysargv.py rename to tests/test_sysargv.py diff --git a/src/tests/test_thread.py b/tests/test_thread.py similarity index 100% rename from src/tests/test_thread.py rename to tests/test_thread.py diff --git a/src/tests/tests.pyproj b/tests/tests.pyproj similarity index 100% rename from src/tests/tests.pyproj rename to tests/tests.pyproj diff --git a/src/tests/utils.py b/tests/utils.py similarity index 100% rename from src/tests/utils.py rename to tests/utils.py From 67032ea055ed6a6e9ea02a5cf2cb8598961e20d7 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 12:03:37 +0100 Subject: [PATCH 049/151] Vendor System.Drawing.Point for testing on .NET Core --- src/testing/arraytest.cs | 12 ++++++++++++ tests/fixtures/.gitkeep | 0 tests/test_array.py | 4 +--- tests/test_class.py | 4 +--- 4 files changed, 14 insertions(+), 6 deletions(-) delete mode 100644 tests/fixtures/.gitkeep diff --git a/src/testing/arraytest.cs b/src/testing/arraytest.cs index 946684962..a3c94e019 100644 --- a/src/testing/arraytest.cs +++ b/src/testing/arraytest.cs @@ -314,4 +314,16 @@ public static Spam[][] EchoRangeAA(Spam[][] items) return items; } } + + public struct Point + { + public Point(float x, float y) + { + X = x; + Y = y; + } + + public float X { get; set; } + public float Y { get; set; } + } } diff --git a/tests/fixtures/.gitkeep b/tests/fixtures/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_array.py b/tests/test_array.py index 232c89ac7..2b1a289ad 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1166,10 +1166,8 @@ def test_boxed_value_type_mutation_result(): # to accidentally write code like the following which is not really # mutating value types in-place but changing boxed copies. - clr.AddReference('System.Drawing') - - from System.Drawing import Point from System import Array + from Python.Test import Point items = Array.CreateInstance(Point, 5) diff --git a/tests/test_class.py b/tests/test_class.py index 4666631f7..f961b3975 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -126,9 +126,7 @@ def __init__(self, v): def test_struct_construction(): """Test construction of structs.""" - clr.AddReference('System.Drawing') - - from System.Drawing import Point + from Python.Test import Point p = Point() assert p.X == 0 From 8bc458b370822ae86038862ab24b0c2663e7a9a2 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 12:09:46 +0100 Subject: [PATCH 050/151] Use approximate comparison for single max/min value comparison --- tests/test_conversion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_conversion.py b/tests/test_conversion.py index 3b290b947..aea95e164 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -406,8 +406,8 @@ def test_uint64_conversion(): def test_single_conversion(): """Test single conversion.""" - assert System.Single.MaxValue == 3.402823e38 - assert System.Single.MinValue == -3.402823e38 + assert System.Single.MaxValue == pytest.approx(3.402823e38) + assert System.Single.MinValue == pytest.approx(-3.402823e38) ob = ConversionTest() assert ob.SingleField == 0.0 From d46fa1e6aa00a291782584018911ee787b0044e5 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 12:39:51 +0100 Subject: [PATCH 051/151] Adjust the import tests to use only commonly available deps --- tests/conftest.py | 4 ---- tests/test_module.py | 47 ++++++++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index aa57f2a1f..3f9436dd9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,10 +66,6 @@ def pytest_configure(config): import clr clr.AddReference("Python.Test") - clr.AddReference("System") - clr.AddReference("System.Collections") - clr.AddReference("System.Data") - clr.AddReference("System.Xml") def pytest_unconfigure(config): diff --git a/tests/test_module.py b/tests/test_module.py index dcdb0248e..d0378e91e 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -30,7 +30,7 @@ def test_import_clr(): def test_version_clr(): import clr - assert clr.__version__ >= "2.2.0" + assert clr.__version__ >= "3.0.0" def test_preload_var(): @@ -111,12 +111,13 @@ def test_dotted_name_import(): def test_multiple_dotted_name_import(): """Test an import bug with multiple dotted imports.""" - import System.Data - assert is_clr_module(System.Data) - assert System.Data.__name__ == 'System.Data' - import System.Data - assert is_clr_module(System.Data) - assert System.Data.__name__ == 'System.Data' + + import System.Reflection + assert is_clr_module(System.Reflection) + assert System.Reflection.__name__ == 'System.Reflection' + import System.Reflection + assert is_clr_module(System.Reflection) + assert System.Reflection.__name__ == 'System.Reflection' def test_dotted_name_import_with_alias(): @@ -192,11 +193,11 @@ def test_dotted_name_import_from_with_alias(): def test_from_module_import_star(): """Test from module import * behavior.""" - clr.AddReference('System.Xml') - + clr.AddReference("System") + count = len(locals().keys()) - m = __import__('System.Xml', globals(), locals(), ['*']) - assert m.__name__ == 'System.Xml' + m = __import__('System', globals(), locals(), ['*']) + assert m.__name__ == 'System' assert is_clr_module(m) assert len(locals().keys()) > count + 1 @@ -212,7 +213,11 @@ def test_implicit_assembly_load(): import Microsoft.Build with warnings.catch_warnings(record=True) as w: - clr.AddReference("System.Windows.Forms") + try: + clr.AddReference("System.Windows.Forms") + except Exception: + pytest.skip() + import System.Windows.Forms as Forms assert is_clr_module(Forms) assert Forms.__name__ == 'System.Windows.Forms' @@ -227,11 +232,11 @@ def test_explicit_assembly_load(): from System.Reflection import Assembly import System, sys - assembly = Assembly.LoadWithPartialName('System.Data') + assembly = Assembly.LoadWithPartialName('System.Runtime') assert assembly is not None - import System.Data - assert 'System.Data' in sys.modules + import System.Runtime + assert 'System.Runtime' in sys.modules assembly = Assembly.LoadWithPartialName('SpamSpamSpamSpamEggsAndSpam') assert assembly is None @@ -275,12 +280,14 @@ def test_module_lookup_recursion(): def test_module_get_attr(): """Test module getattr behavior.""" + import System + import System.Runtime int_type = System.Int32 assert is_clr_class(int_type) - module = System.Xml + module = System.Runtime assert is_clr_module(module) with pytest.raises(AttributeError): @@ -324,7 +331,6 @@ def test_clr_list_assemblies(): from clr import ListAssemblies verbose = list(ListAssemblies(True)) short = list(ListAssemblies(False)) - assert u'mscorlib' in short assert u'System' in short assert u'Culture=' in verbose[0] assert u'Version=' in verbose[0] @@ -377,8 +383,11 @@ def test_assembly_load_thread_safety(): _ = Dictionary[Guid, DateTime]() ModuleTest.JoinThreads() +@pytest.mark.skipif() def test_assembly_load_recursion_bug(): """Test fix for recursion bug documented in #627""" - from System.Configuration import ConfigurationManager - content = dir(ConfigurationManager) + sys_config = pytest.importorskip( + "System.Configuration", reason="System.Configuration can't be imported" + ) + content = dir(sys_config.ConfigurationManager) assert len(content) > 5, content From f0011a51fd494740f9768caf2074e5275ae7a0d1 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 12:47:20 +0100 Subject: [PATCH 052/151] Fix PythonTestRunner to work with new pytest setup --- .../Python.PythonTestsRunner.csproj | 1 + src/python_tests_runner/PythonTestRunner.cs | 3 ++- tests/conftest.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/python_tests_runner/Python.PythonTestsRunner.csproj b/src/python_tests_runner/Python.PythonTestsRunner.csproj index 2d6544614..1006b2148 100644 --- a/src/python_tests_runner/Python.PythonTestsRunner.csproj +++ b/src/python_tests_runner/Python.PythonTestsRunner.csproj @@ -6,6 +6,7 @@ + diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 79b15700e..36e8049d4 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -8,6 +8,7 @@ using NUnit.Framework; using Python.Runtime; +using Python.Test; namespace Python.PythonTestsRunner { @@ -50,7 +51,7 @@ public void RunPythonTest(string testFile, string testName) { folder = Path.GetDirectoryName(folder); } - folder = Path.Combine(folder, "tests"); + folder = Path.Combine(folder, "..", "tests"); string path = Path.Combine(folder, testFile + ".py"); if (!File.Exists(path)) throw new FileNotFoundException("Cannot find test file", path); diff --git a/tests/conftest.py b/tests/conftest.py index 3f9436dd9..cf3341f01 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,12 @@ def pytest_addoption(parser): def pytest_configure(config): global bin_path + if "clr" in sys.modules: + # Already loaded (e.g. by the C# test runner), skip build + import clr + clr.AddReference("Python.Test") + return + runtime_opt = config.getoption("runtime") test_proj_path = os.path.join(cwd, "..", "src", "testing") @@ -70,7 +76,10 @@ def pytest_configure(config): def pytest_unconfigure(config): global bin_path - shutil.rmtree(bin_path) + try: + shutil.rmtree(bin_path) + except Exception: + pass def pytest_report_header(config): """Generate extra report headers""" From c1a01b72fad0f54e87edb2432ad00c135728150c Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 15:21:27 +0100 Subject: [PATCH 053/151] Drop references to the obsolete call --- pythonnet.sln | 1 - src/runtime/native/ABI.cs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pythonnet.sln b/pythonnet.sln index 30f4fd344..e02948c18 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -53,7 +53,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PythonTestsRunner", EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{142A6752-C2C2-4F95-B982-193418001B65}" ProjectSection(SolutionItems) = preProject - configured.props = configured.props Directory.Build.props = Directory.Build.props EndProjectSection EndProject diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index e95b259c5..3264531de 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -24,8 +24,7 @@ internal static void Initialize(Version version, BorrowedReference pyType) if (typeOffsetsClass is null) { var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset")); - string message = $"Searching for {className}, found {string.Join(",", types)}. " + - "If you are building Python.NET from source, make sure you have run 'python setup.py configure' to fill in configured.props"; + string message = $"Searching for {className}, found {string.Join(",", types)}."; throw new NotSupportedException($"Python ABI v{version} is not supported: {message}"); } var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); From fb083bbe0e4fdc5ca6033ee9a83877b70d56c4fe Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 14 Feb 2021 17:05:29 +0100 Subject: [PATCH 054/151] Adjust setup.py for wheel building --- setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ffa18902e..c74ca2c8c 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def run(self): # Add build_dotnet to the build tasks: from distutils.command.build import build as _build from setuptools.command.develop import develop as _develop +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools import Distribution import setuptools @@ -111,6 +112,14 @@ def install_for_development(self): return super().install_for_development() +class bdist_wheel(_bdist_wheel): + def finalize_options(self): + # Monkey patch bdist_wheel to think the package is pure even though we + # include DLLs + super().finalize_options() + self.root_is_pure = True + + # Monkey-patch Distribution s.t. it supports the dotnet_libs attribute Distribution.dotnet_libs = None @@ -118,6 +127,7 @@ def install_for_development(self): "build": build, "build_dotnet": build_dotnet, "develop": develop, + "bdist_wheel": bdist_wheel, } @@ -142,9 +152,8 @@ def install_for_development(self): author="The Contributors of the Python.NET Project", author_email="pythonnet@python.org", packages=["pythonnet", "pythonnet.find_libpython"], - install_requires=["pycparser", "clr_loader"], + install_requires=["clr_loader"], long_description=long_description, - # data_files=[("{install_platlib}", ["{build_lib}/pythonnet"])], py_modules=["clr"], dotnet_libs=dotnet_libs, classifiers=[ @@ -156,6 +165,7 @@ def install_for_development(self): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", From 4c05417dd0e583931176d624ab1bcc86519ae5bb Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 17 Feb 2021 14:12:10 -0800 Subject: [PATCH 055/151] enable manual preview release --- .github/workflows/nuget-preview.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml index 951529452..0f42399e7 100644 --- a/.github/workflows/nuget-preview.yml +++ b/.github/workflows/nuget-preview.yml @@ -3,6 +3,7 @@ name: GitHub Actions on: schedule: - cron: "5 4 3 */1 *" # once a month, at 4:05 on 3rd + workflow_dispatch: jobs: release: From 132b666fb3c4173a3c51e7c5c70dd08437f80c0a Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 17 Feb 2021 14:14:37 -0800 Subject: [PATCH 056/151] rename nuget preview release workflow --- .github/workflows/nuget-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml index 0f42399e7..7339b479b 100644 --- a/.github/workflows/nuget-preview.yml +++ b/.github/workflows/nuget-preview.yml @@ -1,4 +1,4 @@ -name: GitHub Actions +name: NuGet Preview Release on: schedule: From 0f5e7814c68b846cd89019f2e23fed69eaa59eca Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 17 Feb 2021 14:45:43 -0800 Subject: [PATCH 057/151] NuGet Preview pipeline changes missed in https://github.com/pythonnet/pythonnet/pull/1373 --- .github/workflows/nuget-preview.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml index 7339b479b..fcd8ca06e 100644 --- a/.github/workflows/nuget-preview.yml +++ b/.github/workflows/nuget-preview.yml @@ -38,7 +38,6 @@ jobs: - name: Build and Install run: | - python setup.py configure pip install -v . - name: Python Tests From 1ab9cb16363e462993be84124a03b57a13103478 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 22 Jan 2021 22:17:35 -0800 Subject: [PATCH 058/151] simplify PyScope by delegating ownership to PyObject instance --- src/runtime/pyobject.cs | 1 + src/runtime/pyscope.cs | 37 +++++++++---------------------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 902d8c734..3c7f13ec6 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -157,6 +157,7 @@ public T As() return (T)AsManagedObject(typeof(T)); } + internal bool IsDisposed => obj == IntPtr.Zero; /// /// Dispose Method diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 72cb9f247..9d68b76fa 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -29,18 +29,15 @@ public class PyScope : DynamicObject, IDisposable /// /// the python Module object the scope associated with. /// - internal IntPtr obj; - internal BorrowedReference Reference => new BorrowedReference(obj); + readonly PyObject obj; + internal BorrowedReference Reference => obj.Reference; /// - /// the variable dict of the scope. + /// the variable dict of the scope. Borrowed. /// internal readonly IntPtr variables; internal BorrowedReference VarsRef => new BorrowedReference(variables); - private bool _isDisposed; - private bool _finalized = false; - /// /// The Manager this scope associated with. /// It provides scopes this scope can import. @@ -65,7 +62,7 @@ internal PyScope(ref NewReference ptr, PyScopeManager manager) throw new PyScopeException("object is not a module"); } Manager = manager ?? PyScopeManager.Global; - obj = ptr.DangerousMoveToPointer(); + obj = ptr.MoveToPyObject(); //Refcount of the variables not increase variables = Runtime.PyModule_GetDict(Reference).DangerousGetAddress(); PythonException.ThrowIfIsNull(variables); @@ -81,7 +78,6 @@ internal PyScope(ref NewReference ptr, PyScopeManager manager) /// /// return the variable dict of the scope. /// - /// public PyDict Variables() { Runtime.XIncref(variables); @@ -136,7 +132,7 @@ public dynamic Import(string name, string asname = null) /// public void Import(PyScope scope, string asname) { - this.Set(asname, scope.obj); + this.SetPyValue(asname, scope.obj.Handle); } /// @@ -335,11 +331,11 @@ private void Exec(string code, BorrowedReference _globals, BorrowedReference _lo public void Set(string name, object value) { IntPtr _value = Converter.ToPython(value, value?.GetType()); - Set(name, _value); + SetPyValue(name, _value); Runtime.XDecref(_value); } - private void Set(string name, IntPtr value) + private void SetPyValue(string name, IntPtr value) { Check(); using (var pyKey = new PyString(name)) @@ -507,7 +503,7 @@ public override bool TrySetMember(SetMemberBinder binder, object value) private void Check() { - if (_isDisposed) + if (this.obj.IsDisposed) { throw new PyScopeException($"The scope of name '{Name}' object has been disposed"); } @@ -515,23 +511,8 @@ private void Check() public void Dispose() { - if (_isDisposed) - { - return; - } - _isDisposed = true; - Runtime.XDecref(obj); this.OnDispose?.Invoke(this); - } - - ~PyScope() - { - if (_finalized || _isDisposed) - { - return; - } - _finalized = true; - Finalizer.Instance.AddFinalizedObject(ref obj); + this.obj.Dispose(); } } From 707ef36de794ff1a6c7bf7a872cfc43bfa7a5f02 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Thu, 18 Feb 2021 17:03:54 -0600 Subject: [PATCH 059/151] Lossless encoders for IList, IEnumerable and ICollection (#1084) --- src/embed_tests/Codecs.cs | 250 ++++++++++++++++-- src/runtime/Codecs/IterableDecoder.cs | 55 ++++ src/runtime/Codecs/ListDecoder.cs | 50 ++++ src/runtime/Codecs/SequenceDecoder.cs | 52 ++++ .../CollectionWrappers/IterableWrapper.cs | 43 +++ src/runtime/CollectionWrappers/ListWrapper.cs | 57 ++++ .../CollectionWrappers/SequenceWrapper.cs | 113 ++++++++ src/testing/CodecTest.cs | 47 ++++ src/testing/Python.Test.csproj | 1 - src/tests/test_codec.py | 70 +++++ tests/tests.pyproj | 1 + 11 files changed, 720 insertions(+), 19 deletions(-) create mode 100644 src/runtime/Codecs/IterableDecoder.cs create mode 100644 src/runtime/Codecs/ListDecoder.cs create mode 100644 src/runtime/Codecs/SequenceDecoder.cs create mode 100644 src/runtime/CollectionWrappers/IterableWrapper.cs create mode 100644 src/runtime/CollectionWrappers/ListWrapper.cs create mode 100644 src/runtime/CollectionWrappers/SequenceWrapper.cs create mode 100644 src/testing/CodecTest.cs create mode 100644 src/tests/test_codec.py diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 18fcd32d1..266badb9e 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -1,33 +1,39 @@ namespace Python.EmbeddingTest { using System; using System.Collections.Generic; - using System.Text; + using System.Linq; using NUnit.Framework; using Python.Runtime; using Python.Runtime.Codecs; - public class Codecs { + public class Codecs + { [SetUp] - public void SetUp() { + public void SetUp() + { PythonEngine.Initialize(); } [TearDown] - public void Dispose() { + public void Dispose() + { PythonEngine.Shutdown(); } [Test] - public void ConversionsGeneric() { - ConversionsGeneric, ValueTuple>(); + public void TupleConversionsGeneric() + { + TupleConversionsGeneric, ValueTuple>(); } - static void ConversionsGeneric() { + static void TupleConversionsGeneric() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(T value) => restored = value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -38,15 +44,18 @@ static void ConversionsGeneric() { } [Test] - public void ConversionsObject() { - ConversionsObject, ValueTuple>(); + public void TupleConversionsObject() + { + TupleConversionsObject, ValueTuple>(); } - static void ConversionsObject() { + static void TupleConversionsObject() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(object value) => restored = (T)value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -57,12 +66,15 @@ static void ConversionsObject() { } [Test] - public void TupleRoundtripObject() { + public void TupleRoundtripObject() + { TupleRoundtripObject, ValueTuple>(); } - static void TupleRoundtripObject() { + static void TupleRoundtripObject() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); Assert.AreEqual(expected: tuple, actual: restored); @@ -70,18 +82,220 @@ static void TupleRoundtripObject() { } [Test] - public void TupleRoundtripGeneric() { + public void TupleRoundtripGeneric() + { TupleRoundtripGeneric, ValueTuple>(); } - static void TupleRoundtripGeneric() { + static void TupleRoundtripGeneric() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); Assert.AreEqual(expected: tuple, actual: restored); } } + + static PyObject GetPythonIterable() + { + using (Py.GIL()) + { + return PythonEngine.Eval("map(lambda x: x, [1,2,3])"); + } + } + + [Test] + public void ListDecoderTest() + { + var codec = ListDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + + var pyListType = pyList.GetPythonType(); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //we'd have to copy into a list instance to do this, it would not be lossless. + //lossy converters can be implemented outside of the python.net core library + Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); + + //convert to list of int + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); + CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); + + //convert to list of string. This will not work. + //The ListWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python list can be queried without any conversion, + //the IList will report a Count of 3. + IList stringList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); + Assert.AreEqual(stringList.Count, 3); + Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; }); + + //can't convert python iterable to list (this will require a copy which isn't lossless) + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); + } + + [Test] + public void SequenceDecoderTest() + { + var codec = SequenceDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + //SequenceConverter can only convert to any ICollection + var pyList = new PyList(items.ToArray()); + //it can convert a PyList, since PyList satisfies the python sequence protocol + + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); + + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); + CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); + Assert.AreEqual(3, stringCollection.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + //can't convert python iterable to collection (this will require a copy which isn't lossless) + //python iterables do not satisfy the python sequence protocol + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); + + //python tuples do satisfy the python sequence protocol + var pyTuple = new PyTuple(items.ToArray()); + var pyTupleType = pyTuple.GetPythonType(); + + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); + CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); + Assert.AreEqual(3, stringCollection2.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection2.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + } + + [Test] + public void IterableDecoderTest() + { + var codec = IterableDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + var pyListType = pyList.GetPythonType(); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); + CollectionAssert.AreEqual(plainEnumerable1, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will lead to an empty iterable when decoding. TODO - should it throw? + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); + CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); + + Assert.Throws(typeof(InvalidCastException), () => { + foreach (string item in stringEnumerable) + { + var x = item; + } + }); + Assert.Throws(typeof(InvalidCastException), () => { + stringEnumerable.Count(); + }); + + Runtime.CheckExceptionOccurred(); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); + CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + } } /// diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs new file mode 100644 index 000000000..346057238 --- /dev/null +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class IterableDecoder : IPyObjectDecoder + { + internal static bool IsIterable(Type targetType) + { + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return true; + + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IEnumerable<>); + } + + internal static bool IsIterable(PyObject objectType) + { + return objectType.HasAttr("__iter__"); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsIterable(objectType) && IsIterable(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + //first see if T is a plan IEnumerable + if (typeof(T) == typeof(System.Collections.IEnumerable)) + { + object enumerable = new CollectionWrappers.IterableWrapper(pyObj); + value = (T)enumerable; + return true; + } + + var elementType = typeof(T).GetGenericArguments()[0]; + var collectionType = typeof(CollectionWrappers.IterableWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static IterableDecoder Instance { get; } = new IterableDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs new file mode 100644 index 000000000..013f3f3f9 --- /dev/null +++ b/src/runtime/Codecs/ListDecoder.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class ListDecoder : IPyObjectDecoder + { + private static bool IsList(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IList<>); + } + + private static bool IsList(PyObject objectType) + { + //TODO accept any python object that implements the sequence and list protocols + //must implement sequence protocol to fully implement list protocol + //if (!SequenceDecoder.IsSequence(objectType)) return false; + + //returns wheter the type is a list. + return objectType.Handle == Runtime.PyListType; + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsList(objectType) && IsList(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.ListWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static ListDecoder Instance { get; } = new ListDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs new file mode 100644 index 000000000..dce08fd99 --- /dev/null +++ b/src/runtime/Codecs/SequenceDecoder.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class SequenceDecoder : IPyObjectDecoder + { + internal static bool IsSequence(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(ICollection<>); + } + + internal static bool IsSequence(PyObject objectType) + { + //must implement iterable protocol to fully implement sequence protocol + if (!IterableDecoder.IsIterable(objectType)) return false; + + //returns wheter it implements the sequence protocol + //according to python doc this needs to exclude dict subclasses + //but I don't know how to look for that given the objectType + //rather than the instance. + return objectType.HasAttr("__getitem__"); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsSequence(objectType) && IsSequence(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.SequenceWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static SequenceDecoder Instance { get; } = new SequenceDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs new file mode 100644 index 000000000..97979b59b --- /dev/null +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Collections; + +namespace Python.Runtime.CollectionWrappers +{ + internal class IterableWrapper : IEnumerable + { + protected readonly PyObject pyObject; + + public IterableWrapper(PyObject pyObj) + { + if (pyObj == null) + throw new ArgumentNullException(); + pyObject = new PyObject(pyObj.Reference); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public IEnumerator GetEnumerator() + { + PyObject iterObject = null; + using (Py.GIL()) + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); + + while (true) + { + using (Py.GIL()) + { + var item = Runtime.PyIter_Next(iterObject.Handle); + if (item == IntPtr.Zero) + { + Runtime.CheckExceptionOccurred(); + iterObject.Dispose(); + break; + } + + yield return (T)new PyObject(item).AsManagedObject(typeof(T)); + } + } + } + } +} diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs new file mode 100644 index 000000000..ec2476370 --- /dev/null +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class ListWrapper : SequenceWrapper, IList + { + public ListWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public T this[int index] + { + get + { + var item = Runtime.PyList_GetItem(pyObject.Reference, index); + var pyItem = new PyObject(item); + return pyItem.As(); + } + set + { + var pyItem = value.ToPython(); + var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem.Handle); + if (result == -1) + Runtime.CheckExceptionOccurred(); + } + } + + public int IndexOf(T item) + { + return indexOf(item); + } + + public void Insert(int index, T item) + { + if (IsReadOnly) + throw new InvalidOperationException("Collection is read-only"); + + var pyItem = item.ToPython(); + + var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem.Handle); + if (result == -1) + Runtime.CheckExceptionOccurred(); + } + + public void RemoveAt(int index) + { + var result = removeAt(index); + + //PySequence_DelItem will set an error if it fails. throw it here + //since RemoveAt does not have a bool return value. + if (result == false) + Runtime.CheckExceptionOccurred(); + } + } +} diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs new file mode 100644 index 000000000..945019850 --- /dev/null +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class SequenceWrapper : IterableWrapper, ICollection + { + public SequenceWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public int Count + { + get + { + var size = Runtime.PySequence_Size(pyObject.Reference); + if (size == -1) + { + Runtime.CheckExceptionOccurred(); + } + + return (int)size; + } + } + + public virtual bool IsReadOnly => false; + + public virtual void Add(T item) + { + //not implemented for Python sequence. + //ICollection is the closest analogue but it doesn't map perfectly. + //SequenceWrapper is not final and can be subclassed if necessary + throw new NotImplementedException(); + } + + public void Clear() + { + if (IsReadOnly) + throw new NotImplementedException(); + var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); + if (result == -1) + { + Runtime.CheckExceptionOccurred(); + } + } + + public bool Contains(T item) + { + //not sure if IEquatable is implemented and this will work! + foreach (var element in this) + if (element.Equals(item)) return true; + + return false; + } + + public void CopyTo(T[] array, int arrayIndex) + { + if (array == null) + throw new NullReferenceException(); + + if ((array.Length - arrayIndex) < this.Count) + throw new InvalidOperationException("Attempting to copy to an array that is too small"); + + var index = 0; + foreach (var item in this) + { + array[index + arrayIndex] = item; + index++; + } + } + + protected bool removeAt(int index) + { + if (IsReadOnly) + throw new NotImplementedException(); + if (index >= Count || index < 0) + return false; + + var result = Runtime.PySequence_DelItem(pyObject.Handle, index); + + if (result == 0) + return true; + + Runtime.CheckExceptionOccurred(); + return false; + } + + protected int indexOf(T item) + { + var index = 0; + foreach (var element in this) + { + if (element.Equals(item)) return index; + index++; + } + + return -1; + } + + public bool Remove(T item) + { + var result = removeAt(indexOf(item)); + + //clear the python exception from PySequence_DelItem + //it is idiomatic in C# to return a bool rather than + //throw for a failed Remove in ICollection + if (result == false) + Runtime.PyErr_Clear(); + return result; + } + } +} diff --git a/src/testing/CodecTest.cs b/src/testing/CodecTest.cs new file mode 100644 index 000000000..74f77531b --- /dev/null +++ b/src/testing/CodecTest.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Test +{ + public class ListMember + { + public ListMember(int value, string name) + { + Value = value; + Name = name; + } + + public int Value { get; set; } + public string Name { get; set; } + } + + public class ListConversionTester + { + public int GetLength(IEnumerable o) + { + return o.Count(); + } + public int GetLength(ICollection o) + { + return o.Count; + } + public int GetLength(IList o) + { + return o.Count; + } + public int GetLength2(IEnumerable o) + { + return o.Count(); + } + public int GetLength2(ICollection o) + { + return o.Count; + } + public int GetLength2(IList o) + { + return o.Count; + } + } +} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 4b7e4d93b..f7bc10bb4 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -4,7 +4,6 @@ true true - diff --git a/src/tests/test_codec.py b/src/tests/test_codec.py new file mode 100644 index 000000000..9fdbfbbf5 --- /dev/null +++ b/src/tests/test_codec.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +"""Test conversions using codecs from client python code""" +import clr +import System +import pytest +import Python.Runtime +from Python.Test import ListConversionTester, ListMember + +class int_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter + +class obj_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return ListMember(self.counter, "Number " + str(self.counter)) + +def test_iterable(): + """Test that a python iterable can be passed into a function that takes an IEnumerable""" + + #Python.Runtime.Codecs.ListDecoder.Register() + #Python.Runtime.Codecs.SequenceDecoder.Register() + Python.Runtime.Codecs.IterableDecoder.Register() + ob = ListConversionTester() + + iterable = int_iterable() + assert 3 == ob.GetLength(iterable) + + iterable2 = obj_iterable() + assert 3 == ob.GetLength2(iterable2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_sequence(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + tup = (1,2,3) + assert 3 == ob.GetLength(tup) + + tup2 = (ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")) + assert 3 == ob.GetLength(tup2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_list(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + l = [1,2,3] + assert 3 == ob.GetLength(l) + + l2 = [ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")] + assert 3 == ob.GetLength(l2) + + Python.Runtime.PyObjectConversions.Reset() diff --git a/tests/tests.pyproj b/tests/tests.pyproj index 439bc856a..fc1053f7b 100644 --- a/tests/tests.pyproj +++ b/tests/tests.pyproj @@ -52,6 +52,7 @@ + From d0d7616fdf184e419330c208faa0bc87cedfdb49 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sat, 20 Feb 2021 19:26:46 -0800 Subject: [PATCH 060/151] ensure interned strings can not be referenced after InternString.Shutdown --- src/runtime/intern.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/intern.cs b/src/runtime/intern.cs index d8bdf863e..ced1e5e92 100644 --- a/src/runtime/intern.cs +++ b/src/runtime/intern.cs @@ -42,9 +42,10 @@ public static void Initialize() public static void Shutdown() { - foreach (var ptr in _intern2strings.Keys) + foreach (var entry in _intern2strings) { - Runtime.XDecref(ptr); + Runtime.XDecref(entry.Key); + typeof(PyIdentifier).GetField(entry.Value).SetValue(null, IntPtr.Zero); } _string2interns = null; _intern2strings = null; From 83328d2d3f4597db608e4ad54422264728a3ef46 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sat, 20 Feb 2021 18:46:38 -0800 Subject: [PATCH 061/151] made InterruptTest more robust should resolve this failure: https://github.com/pythonnet/pythonnet/pull/1392/checks?check_run_id=1944113649 related to https://github.com/pythonnet/pythonnet/pull/1337 --- src/embed_tests/TestInterrupt.cs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/TestInterrupt.cs b/src/embed_tests/TestInterrupt.cs index e075d1194..a40407782 100644 --- a/src/embed_tests/TestInterrupt.cs +++ b/src/embed_tests/TestInterrupt.cs @@ -1,5 +1,5 @@ - using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -30,14 +30,13 @@ public void Dispose() [Test] public void InterruptTest() { - int runSimpleStringReturnValue = int.MinValue; - ulong pythonThreadID = ulong.MinValue; - Task.Factory.StartNew(() => + long pythonThreadID = 0; + var asyncCall = Task.Factory.StartNew(() => { using (Py.GIL()) { - pythonThreadID = PythonEngine.GetPythonThreadID(); - runSimpleStringReturnValue = PythonEngine.RunSimpleString(@" + Interlocked.Exchange(ref pythonThreadID, (long)PythonEngine.GetPythonThreadID()); + return PythonEngine.RunSimpleString(@" import time while True: @@ -45,19 +44,21 @@ import time } }); - Thread.Sleep(200); - - Assert.AreNotEqual(ulong.MinValue, pythonThreadID); + var timeout = Stopwatch.StartNew(); + while (Interlocked.Read(ref pythonThreadID) == 0) + { + Assert.Less(timeout.Elapsed, TimeSpan.FromSeconds(5), "thread ID was not assigned in time"); + } using (Py.GIL()) { - int interruptReturnValue = PythonEngine.Interrupt(pythonThreadID); + int interruptReturnValue = PythonEngine.Interrupt((ulong)Interlocked.Read(ref pythonThreadID)); Assert.AreEqual(1, interruptReturnValue); } - Thread.Sleep(300); + Assert.IsTrue(asyncCall.Wait(TimeSpan.FromSeconds(5)), "Async thread was not interrupted in time"); - Assert.AreEqual(-1, runSimpleStringReturnValue); + Assert.AreEqual(-1, asyncCall.Result); } } } From 6db4373cc745d697f1249dc8248e88a646addbef Mon Sep 17 00:00:00 2001 From: Victor Date: Sat, 20 Feb 2021 19:54:23 -0800 Subject: [PATCH 062/151] added NuGet shields to README --- README.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a03468e7d..afe988105 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,11 @@ pythonnet - Python.NET |gh shield| |appveyor shield| -|license shield| |pypi package version| |conda-forge version| |python supported shield| +|license shield| + +|pypi package version| |conda-forge version| |python supported shield| + +|nuget preview shield| |nuget release shield| Python.NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and @@ -130,5 +134,9 @@ This project is supported by the `.NET Foundation :target: http://stackoverflow.com/questions/tagged/python.net .. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg :target: https://anaconda.org/conda-forge/pythonnet +.. |nuget preview shield| image:: https://img.shields.io/nuget/vpre/pythonnet + :target: https://www.nuget.org/packages/pythonnet/ +.. |nuget release shield| image:: https://img.shields.io/nuget/v/pythonnet + :target: https://www.nuget.org/packages/pythonnet/ .. |gh shield| image:: https://github.com/pythonnet/pythonnet/workflows/GitHub%20Actions/badge.svg :target: https://github.com/pythonnet/pythonnet/actions?query=branch%3Amaster From 6f1219f7d3f6a53c9ae932e51cb2c15aad54fb54 Mon Sep 17 00:00:00 2001 From: Victor Date: Sat, 20 Feb 2021 21:25:57 -0800 Subject: [PATCH 063/151] mentioned PythonDLL in README --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index afe988105..7c2b27b93 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,8 @@ module: Embedding Python in .NET ------------------------ +- You must set `Runtime.PythonDLL` or `PYTHONNET_PYDLL` environment variable + starting with version 3.0, otherwise you will receive `TypeInitializationException`. - All calls to python should be inside a ``using (Py.GIL()) {/* Your code here */}`` block. - Import python modules using ``dynamic mod = Py.Import("mod")``, then From c4f2ad77d2102515733762e248b8b296f5f3b509 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 22 Feb 2021 19:41:43 +0100 Subject: [PATCH 064/151] Adjust static libpython detection (#1396) * Check for Py_ENABLE_SHARED on Unix --- pythonnet/find_libpython/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pythonnet/find_libpython/__init__.py b/pythonnet/find_libpython/__init__.py index 185540c8f..3ae28970e 100644 --- a/pythonnet/find_libpython/__init__.py +++ b/pythonnet/find_libpython/__init__.py @@ -77,6 +77,9 @@ class Dl_info(ctypes.Structure): def _linked_libpython_unix(): + if not sysconfig.get_config_var("Py_ENABLE_SHARED"): + return None + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] libdl.dladdr.restype = ctypes.c_int @@ -88,8 +91,6 @@ def _linked_libpython_unix(): if retcode == 0: # means error return None path = os.path.realpath(dlinfo.dli_fname.decode()) - if path == os.path.realpath(sys.executable): - return None return path From c3fc7f08a0137c560840a0ebe2f2b401a83b3739 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 19 Feb 2021 12:52:50 -0800 Subject: [PATCH 065/151] allow comparing BorrowedReference to null --- src/runtime/BorrowedReference.cs | 8 ++++++++ src/runtime/importhook.cs | 6 +++--- src/runtime/tricks/NullOnly.cs | 12 ++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 src/runtime/tricks/NullOnly.cs diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 2f5c347c7..d49f52fe9 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -28,6 +28,14 @@ public BorrowedReference(IntPtr pointer) => a.pointer == b.pointer; public static bool operator !=(BorrowedReference a, BorrowedReference b) => a.pointer != b.pointer; + public static bool operator ==(BorrowedReference reference, NullOnly @null) + => reference.IsNull; + public static bool operator !=(BorrowedReference reference, NullOnly @null) + => !reference.IsNull; + public static bool operator ==(NullOnly @null, BorrowedReference reference) + => reference.IsNull; + public static bool operator !=(NullOnly @null, BorrowedReference reference) + => !reference.IsNull; public override bool Equals(object obj) { if (obj is IntPtr ptr) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 066c765fe..13b45df51 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -155,7 +155,7 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa // find any items from the from list and get them from the root if they're not // already in the module dictionary - if (fromList != null && fromList != default) + if (fromList != null) { if (Runtime.PyTuple_Check(fromList)) { @@ -224,7 +224,7 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) if (num_args >= 4) { fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != default && + if (fromList != null && Runtime.PyObject_IsTrue(fromList) == 1) { fromlist = true; @@ -297,7 +297,7 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) BorrowedReference modules = Runtime.PyImport_GetModuleDict(); BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name); - if (module != default) + if (module != null) { if (fromlist) { diff --git a/src/runtime/tricks/NullOnly.cs b/src/runtime/tricks/NullOnly.cs new file mode 100644 index 000000000..cc2679a61 --- /dev/null +++ b/src/runtime/tricks/NullOnly.cs @@ -0,0 +1,12 @@ +namespace Python.Runtime +{ + /// + /// An utility class, that can only have one value: null. + /// Useful for overloading operators on structs, + /// that have meaningful concept of null value (e.g. pointers and references). + /// + class NullOnly + { + private NullOnly() { } + } +} From 60bb68c2fab8bfb271dd4f7bab9e15af2a402e9c Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 19 Feb 2021 12:53:04 -0800 Subject: [PATCH 066/151] removed unused Utf8Marshaler --- src/runtime/CustomMarshaler.cs | 45 ---------------------------------- 1 file changed, 45 deletions(-) diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 4814e6c0b..3ef5cd662 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -189,49 +189,4 @@ public static ICustomMarshaler GetInstance(string cookie) return Instance; } } - - - /// - /// Custom Marshaler to deal with Managed String to Native - /// conversion on UTF-8. Use on functions that expect UTF-8 encoded - /// strings like `PyUnicode_FromStringAndSize` - /// - /// - /// If instead we used `MarshalAs(UnmanagedType.LPWStr)` the output to - /// `foo` would be `f\x00o\x00o\x00`. - /// - internal class Utf8Marshaler : MarshalerBase - { - private static readonly MarshalerBase Instance = new Utf8Marshaler(); - private static readonly Encoding PyEncoding = Encoding.UTF8; - - public override IntPtr MarshalManagedToNative(object managedObj) - { - var s = managedObj as string; - - if (s == null) - { - return IntPtr.Zero; - } - - byte[] bStr = PyEncoding.GetBytes(s + "\0"); - IntPtr mem = Marshal.AllocHGlobal(bStr.Length); - try - { - Marshal.Copy(bStr, 0, mem, bStr.Length); - } - catch (Exception) - { - Marshal.FreeHGlobal(mem); - throw; - } - - return mem; - } - - public static ICustomMarshaler GetInstance(string cookie) - { - return Instance; - } - } } From 50834ba7b50247ab71f57fcc0dd475027a531003 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 19 Feb 2021 13:28:27 -0800 Subject: [PATCH 067/151] introduced PyModule (inherits PyScope) changed PyScope to inherit from PyObject --- src/embed_tests/TestFinalizer.cs | 14 +++++- src/embed_tests/TestPythonException.cs | 4 +- src/embed_tests/TestRuntime.cs | 4 +- src/embed_tests/pyimport.cs | 4 +- src/embed_tests/pyinitialize.cs | 3 +- src/runtime/exceptions.cs | 21 ++++----- src/runtime/pymodule.cs | 41 ++++++++++++++++ src/runtime/pyscope.cs | 51 ++++++++++---------- src/runtime/pythonengine.cs | 65 +++++++------------------- src/runtime/pythonexception.cs | 4 +- src/runtime/runtime.cs | 44 +++++++++-------- src/runtime/runtime_state.cs | 9 ++-- src/testing/threadtest.cs | 4 +- 13 files changed, 145 insertions(+), 123 deletions(-) create mode 100644 src/runtime/pymodule.cs diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 46e2fcdf1..c040e6930 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -101,7 +101,17 @@ public void CollectOnShutdown() PythonEngine.Shutdown(); garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.IsEmpty(garbage); + + if (garbage.Count > 0) + { + PythonEngine.Initialize(); + string objects = string.Join("\n", garbage.Select(ob => + { + var obj = new PyObject(new BorrowedReference(ob)); + return $"{obj} [{obj.GetPythonType()}@{obj.Handle}]"; + })); + Assert.Fail("Garbage is not empty:\n" + objects); + } } [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] // ensure lack of references to obj @@ -173,7 +183,7 @@ public void SimpleTestMemory() bool oldState = Finalizer.Instance.Enable; try { - using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyModule gcModule = PyModule.Import("gc")) using (PyObject pyCollect = gcModule.GetAttr("collect")) { long span1 = CompareWithFinalizerOn(pyCollect, false); diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 31addfba1..a74fc3a8b 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -46,7 +46,7 @@ public void TestPythonErrorTypeName() { try { - var module = PythonEngine.ImportModule("really____unknown___module"); + var module = PyModule.Import("really____unknown___module"); Assert.Fail("Unknown module should not be loaded"); } catch (PythonException ex) @@ -95,7 +95,7 @@ public void TestPythonExceptionFormatNoTraceback() { try { - var module = PythonEngine.ImportModule("really____unknown___module"); + var module = PyModule.Import("really____unknown___module"); Assert.Fail("Unknown module should not be loaded"); } catch (PythonException ex) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 59c66cc5e..32369190c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -96,7 +96,7 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. var threading = Runtime.Runtime.PyImport_ImportModule("threading"); Exceptions.ErrorCheck(threading); - var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading)); + var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); Exceptions.ErrorCheck(threadingDict); var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); if (lockType.IsNull) @@ -110,6 +110,8 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); + threading.Dispose(); + Runtime.Runtime.Py_Finalize(); } } diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index 24f31acff..b34a5d25b 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -52,7 +52,7 @@ public void Dispose() [Test] public void TestDottedName() { - PyObject module = PythonEngine.ImportModule("PyImportTest.test.one"); + var module = PyModule.Import("PyImportTest.test.one"); Assert.IsNotNull(module); } @@ -62,7 +62,7 @@ public void TestDottedName() [Test] public void TestSysArgsImportException() { - PyObject module = PythonEngine.ImportModule("PyImportTest.sysargv"); + var module = PyModule.Import("PyImportTest.sysargv"); Assert.IsNotNull(module); } diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index c774680dd..66a9a3f7c 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -175,7 +175,8 @@ public static void TestRunExitFuncs() { called = true; }; - atexit.InvokeMethod("register", callback.ToPython()); + atexit.InvokeMethod("register", callback.ToPython()).Dispose(); + atexit.Dispose(); Runtime.Runtime.Shutdown(); Assert.True(called); } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index afd0bc14e..da8653853 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -95,8 +95,8 @@ internal static Exception ToException(IntPtr ob) /// public static class Exceptions { - internal static IntPtr warnings_module; - internal static IntPtr exceptions_module; + internal static PyModule warnings_module; + internal static PyModule exceptions_module; /// /// Initialization performed on startup of the Python runtime. @@ -104,15 +104,12 @@ public static class Exceptions internal static void Initialize() { string exceptionsModuleName = "builtins"; - exceptions_module = Runtime.PyImport_ImportModule(exceptionsModuleName); - - Exceptions.ErrorCheck(exceptions_module); - warnings_module = Runtime.PyImport_ImportModule("warnings"); - Exceptions.ErrorCheck(warnings_module); + exceptions_module = PyModule.Import(exceptionsModuleName); + warnings_module = PyModule.Import("warnings"); Type type = typeof(Exceptions); foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { - IntPtr op = Runtime.PyObject_GetAttrString(exceptions_module, fi.Name); + IntPtr op = Runtime.PyObject_GetAttrString(exceptions_module.obj, fi.Name); if (op != IntPtr.Zero) { fi.SetValue(type, op); @@ -147,8 +144,8 @@ internal static void Shutdown() Runtime.XDecref(op); fi.SetValue(null, IntPtr.Zero); } - Runtime.Py_CLEAR(ref exceptions_module); - Runtime.Py_CLEAR(ref warnings_module); + exceptions_module.Dispose(); + warnings_module.Dispose(); } /// @@ -348,9 +345,7 @@ public static void warn(string message, IntPtr exception, int stacklevel) Exceptions.RaiseTypeError("Invalid exception"); } - Runtime.XIncref(warnings_module); - IntPtr warn = Runtime.PyObject_GetAttrString(warnings_module, "warn"); - Runtime.XDecref(warnings_module); + IntPtr warn = Runtime.PyObject_GetAttrString(warnings_module.obj, "warn"); Exceptions.ErrorCheck(warn); IntPtr args = Runtime.PyTuple_New(3); diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs new file mode 100644 index 000000000..800edb686 --- /dev/null +++ b/src/runtime/pymodule.cs @@ -0,0 +1,41 @@ +using System; + +namespace Python.Runtime +{ + public class PyModule : PyScope + { + internal PyModule(ref NewReference reference) : base(ref reference, PyScopeManager.Global) { } + public PyModule(PyObject o) : base(o.Reference, PyScopeManager.Global) { } + + /// + /// Given a module or package name, import the + /// module and return the resulting module object as a . + /// + /// Fully-qualified module or package name + public static PyModule Import(string name) + { + NewReference op = Runtime.PyImport_ImportModule(name); + PythonException.ThrowIfIsNull(op); + return new PyModule(ref op); + } + + /// + /// Reloads the module, and returns the updated object + /// + public PyModule Reload() + { + NewReference op = Runtime.PyImport_ReloadModule(this.Reference); + PythonException.ThrowIfIsNull(op); + return new PyModule(ref op); + } + + public static PyModule FromString(string name, string code) + { + using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); + PythonException.ThrowIfIsNull(c); + NewReference m = Runtime.PyImport_ExecCodeModule(name, c); + PythonException.ThrowIfIsNull(m); + return new PyModule(ref m); + } + } +} diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 9d68b76fa..4d09004bd 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,15 +22,9 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : PyObject { - public readonly string Name; - - /// - /// the python Module object the scope associated with. - /// - readonly PyObject obj; - internal BorrowedReference Reference => obj.Reference; + public string Name { get; } /// /// the variable dict of the scope. Borrowed. @@ -49,20 +43,24 @@ public class PyScope : DynamicObject, IDisposable /// public event Action OnDispose; - /// - /// Constructor - /// - /// - /// Create a scope based on a Python Module. - /// - internal PyScope(ref NewReference ptr, PyScopeManager manager) + /// Create a scope based on a Python Module. + internal PyScope(ref NewReference reference, PyScopeManager manager) + : this(reference.DangerousMoveToPointer(), manager) { } + /// Create a scope based on a Python Module. + internal PyScope(BorrowedReference reference, PyScopeManager manager) + : this(reference.DangerousGetAddress(), manager) { - if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(ptr), Runtime.PyModuleType)) + Runtime.XIncref(reference.DangerousGetAddress()); + } + + /// Create a scope based on a Python Module. + private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr) + { + if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(Reference), Runtime.PyModuleType)) { throw new PyScopeException("object is not a module"); } Manager = manager ?? PyScopeManager.Global; - obj = ptr.MoveToPyObject(); //Refcount of the variables not increase variables = Runtime.PyModule_GetDict(Reference).DangerousGetAddress(); PythonException.ThrowIfIsNull(variables); @@ -72,7 +70,8 @@ internal PyScope(ref NewReference ptr, PyScopeManager manager) Runtime.PyEval_GetBuiltins() ); PythonException.ThrowIfIsNotZero(res); - this.Name = this.Get("__name__"); + using var name = this.Get("__name__"); + this.Name = name.As(); } /// @@ -118,7 +117,7 @@ public dynamic Import(string name, string asname = null) } else { - PyObject module = PythonEngine.ImportModule(name); + var module = PyModule.Import(name); Import(module, asname); return module; } @@ -132,7 +131,7 @@ public dynamic Import(string name, string asname = null) /// public void Import(PyScope scope, string asname) { - this.SetPyValue(asname, scope.obj.Handle); + this.SetPyValue(asname, scope.Handle); } /// @@ -169,7 +168,7 @@ public void ImportAll(string name) } else { - PyObject module = PythonEngine.ImportModule(name); + var module = PyModule.Import(name); ImportAll(module); } } @@ -503,16 +502,20 @@ public override bool TrySetMember(SetMemberBinder binder, object value) private void Check() { - if (this.obj.IsDisposed) + if (this.obj == IntPtr.Zero) { throw new PyScopeException($"The scope of name '{Name}' object has been disposed"); } } - public void Dispose() + protected override void Dispose(bool disposing) { + if (this.obj == IntPtr.Zero) + { + return; + } + base.Dispose(disposing); this.OnDispose?.Invoke(this); - this.obj.Dispose(); } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 35ea3f6d2..b12309733 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -468,62 +468,27 @@ public static void EndAllowThreads(IntPtr ts) Runtime.PyEval_RestoreThread(ts); } + [Obsolete("Use PyModule.Import")] + public static PyObject ImportModule(string name) => PyModule.Import(name); - /// - /// ImportModule Method - /// - /// - /// Given a fully-qualified module or package name, import the - /// module and return the resulting module object as a PyObject - /// or null if an exception is raised. - /// - public static PyObject ImportModule(string name) - { - IntPtr op = Runtime.PyImport_ImportModule(name); - PythonException.ThrowIfIsNull(op); - return new PyObject(op); - } - - - /// - /// ReloadModule Method - /// - /// - /// Given a PyObject representing a previously loaded module, reload - /// the module. - /// + [Obsolete("Use PyModule.Reload")] public static PyObject ReloadModule(PyObject module) - { - IntPtr op = Runtime.PyImport_ReloadModule(module.Handle); - PythonException.ThrowIfIsNull(op); - return new PyObject(op); - } + => module is PyModule pyModule ? pyModule.Reload() : new PyModule(module).Reload(); - - /// - /// ModuleFromString Method - /// - /// - /// Given a string module name and a string containing Python code, - /// execute the code in and return a module of the given name. - /// + [Obsolete("Use PyModule.FromString")] public static PyObject ModuleFromString(string name, string code) - { - IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); - PythonException.ThrowIfIsNull(c); - IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); - PythonException.ThrowIfIsNull(m); - return new PyObject(m); - } + => PyModule.FromString(name, code); + public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) { var flag = (int)mode; - IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); + NewReference ptr = Runtime.Py_CompileString(code, filename, flag); PythonException.ThrowIfIsNull(ptr); - return new PyObject(ptr); + return ptr.MoveToPyObject(); } + /// /// Eval Method /// @@ -742,10 +707,12 @@ public static KeywordArguments kw(params object[] kv) return dict; } - public static PyObject Import(string name) - { - return PythonEngine.ImportModule(name); - } + /// + /// Given a module or package name, import the + /// module and return the resulting module object as a . + /// + /// Fully-qualified module or package name + public static PyModule Import(string name) => PyModule.Import(name); public static void SetArgv() { diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 7dd4f0811..648888293 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -48,7 +48,7 @@ public PythonException() if (_pyTB != IntPtr.Zero) { - using PyObject tb_module = PythonEngine.ImportModule("traceback"); + using var tb_module = PyModule.Import("traceback"); Runtime.XIncref(_pyTB); using var pyTB = new PyObject(_pyTB); @@ -198,7 +198,7 @@ public string Format() using (PyObject pyType = new PyObject(type)) using (PyObject pyValue = new PyObject(value)) using (PyObject pyTB = new PyObject(tb)) - using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + using (PyObject tb_mod = PyModule.Import("traceback")) { var buffer = new StringBuilder(); var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec7f5e446..173bd294d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -278,9 +278,8 @@ private static void InitPyMembers() _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); { - IntPtr sys = PyImport_ImportModule("sys"); - PyModuleType = PyObject_Type(sys); - XDecref(sys); + using var sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys.DangerousMoveToPointer()); } } @@ -937,14 +936,14 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR /// Return value: New reference. /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. /// - internal static IntPtr Py_CompileString(string str, string file, int start) + internal static NewReference Py_CompileString(string str, string file, int start) { using var strPtr = new StrPtr(str, Encoding.UTF8); using var fileObj = new PyString(file); return Delegates.Py_CompileStringObject(strPtr, fileObj.Reference, start, Utf8String, -1); } - internal static IntPtr PyImport_ExecCodeModule(string name, IntPtr code) + internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { using var namePtr = new StrPtr(name, Encoding.UTF8); return Delegates.PyImport_ExecCodeModule(namePtr, code); @@ -1027,11 +1026,18 @@ internal static int PyObject_HasAttrString(BorrowedReference pointer, string nam internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) { using var namePtr = new StrPtr(name, Encoding.UTF8); - return Delegates.PyObject_GetAttrString(pointer, namePtr); + return Delegates.PyObject_GetAttrString(new BorrowedReference(pointer), namePtr) + .DangerousMoveToPointerOrNull(); } + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_GetAttrString(pointer, namePtr); + } - internal static IntPtr PyObject_GetAttrString(IntPtr pointer, StrPtr name) => Delegates.PyObject_GetAttrString(pointer, name); + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, StrPtr name) + => Delegates.PyObject_GetAttrString(pointer, name); internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) @@ -1917,13 +1923,13 @@ internal static string PyModule_GetFilename(IntPtr module) /// Return value: New reference. /// - internal static IntPtr PyImport_ImportModule(string name) + internal static NewReference PyImport_ImportModule(string name) { using var namePtr = new StrPtr(name, Encoding.UTF8); return Delegates.PyImport_ImportModule(namePtr); } - internal static IntPtr PyImport_ReloadModule(IntPtr module) => Delegates.PyImport_ReloadModule(module); + internal static NewReference PyImport_ReloadModule(BorrowedReference module) => Delegates.PyImport_ReloadModule(module); internal static BorrowedReference PyImport_AddModule(string name) @@ -2294,13 +2300,13 @@ static Delegates() PyRun_SimpleStringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_SimpleStringFlags), GetUnmanagedDll(_PythonDll)); PyRun_StringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_StringFlags), GetUnmanagedDll(_PythonDll)); PyEval_EvalCode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_EvalCode), GetUnmanagedDll(_PythonDll)); - Py_CompileStringObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringObject), GetUnmanagedDll(_PythonDll)); - PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); + Py_CompileStringObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringObject), GetUnmanagedDll(_PythonDll)); + PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); PyCFunction_NewEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_NewEx), GetUnmanagedDll(_PythonDll)); PyCFunction_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_Call), GetUnmanagedDll(_PythonDll)); PyMethod_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_New), GetUnmanagedDll(_PythonDll)); PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); - PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); PyObject_GetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttr), GetUnmanagedDll(_PythonDll)); @@ -2458,8 +2464,8 @@ static Delegates() PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); - PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); - PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); + PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); + PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); PyImport_AddModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_AddModule), GetUnmanagedDll(_PythonDll)); PyImport_GetModuleDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_GetModuleDict), GetUnmanagedDll(_PythonDll)); PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); @@ -2566,13 +2572,13 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyRun_SimpleStringFlags { get; } internal static delegate* unmanaged[Cdecl] PyRun_StringFlags { get; } internal static delegate* unmanaged[Cdecl] PyEval_EvalCode { get; } - internal static delegate* unmanaged[Cdecl] Py_CompileStringObject { get; } - internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } + internal static delegate* unmanaged[Cdecl] Py_CompileStringObject { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } internal static delegate* unmanaged[Cdecl] PyCFunction_NewEx { get; } internal static delegate* unmanaged[Cdecl] PyCFunction_Call { get; } internal static delegate* unmanaged[Cdecl] PyMethod_New { get; } internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } - internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetAttr { get; } @@ -2723,8 +2729,8 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } - internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } - internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_AddModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_GetModuleDict { get; } internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 295219675..b541a7c44 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -139,18 +139,15 @@ private static void RestoreObjects(IntPtr dummyGC) public static IEnumerable PyGCGetObjects() { - var gc = PyImport_ImportModule("gc"); - PythonException.ThrowIfIsNull(gc); - var get_objects = PyObject_GetAttrString(gc, "get_objects"); - var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + using var gc = PyModule.Import("gc"); + using var get_objects = gc.GetAttr("get_objects"); + var objs = PyObject_CallObject(get_objects.Handle, IntPtr.Zero); var length = PyList_Size(new BorrowedReference(objs)); for (long i = 0; i < length; i++) { var obj = PyList_GetItem(new BorrowedReference(objs), i); yield return obj.DangerousGetAddress(); } - XDecref(objs); - XDecref(gc); } public static IEnumerable GetModuleNames() diff --git a/src/testing/threadtest.cs b/src/testing/threadtest.cs index 9c76929b2..6664c3643 100644 --- a/src/testing/threadtest.cs +++ b/src/testing/threadtest.cs @@ -34,7 +34,7 @@ public static string CallEchoString(string arg) { if (module == null) { - module = PythonEngine.ModuleFromString("tt", testmod); + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring"); var parg = new PyString(arg); @@ -58,7 +58,7 @@ public static string CallEchoString2(string arg) { if (module == null) { - module = PythonEngine.ModuleFromString("tt", testmod); + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring2"); From ac68aab1374b2553e05e6e4cec87753e83c15f41 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 19 Feb 2021 15:08:59 -0800 Subject: [PATCH 068/151] minor fixes --- src/embed_tests/GlobalTestsSetup.cs | 2 +- src/embed_tests/TestNativeTypeOffset.cs | 2 +- src/runtime/pystring.cs | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs index 458ab6a99..9a832cb0c 100644 --- a/src/embed_tests/GlobalTestsSetup.cs +++ b/src/embed_tests/GlobalTestsSetup.cs @@ -7,7 +7,7 @@ namespace Python.EmbeddingTest // As the SetUpFixture, the OneTimeTearDown of this class is executed after // all tests have run. [SetUpFixture] - public class GlobalTestsSetup + public partial class GlobalTestsSetup { [OneTimeTearDown] public void FinalCleanup() diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 7dd5a765e..8efd16e02 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -9,7 +9,7 @@ using Python.Runtime; -namespace Python.EmbeddingPythonTest +namespace Python.EmbeddingTest { public class TestNativeTypeOffset { diff --git a/src/runtime/pystring.cs b/src/runtime/pystring.cs index b3d0dc86d..07eabba14 100644 --- a/src/runtime/pystring.cs +++ b/src/runtime/pystring.cs @@ -67,11 +67,8 @@ public PyString(string s) : base(FromString(s)) /// - /// IsStringType Method - /// - /// /// Returns true if the given object is a Python string. - /// + /// public static bool IsStringType(PyObject value) { return Runtime.PyString_Check(value.obj); From 6f1064185250ae4080b8d7f3fa491b78425380db Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:55:30 -0800 Subject: [PATCH 069/151] PyImport test validate no initialization failures --- src/embed_tests/pyimport.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index 24f31acff..2823c3d19 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -34,7 +34,9 @@ public void SetUp() TestContext.Out.WriteLine(testPath); IntPtr str = Runtime.Runtime.PyString_FromString(testPath); + Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); + Assert.IsFalse(path.IsNull); Runtime.Runtime.PyList_Append(path, str); Runtime.Runtime.XDecref(str); } From 676fbb42cd904f042ed1387c95c54baf090fbc98 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:56:50 -0800 Subject: [PATCH 070/151] PyObject.Length raises an exception when object does not have a concept of length --- CHANGELOG.md | 1 + src/runtime/pyobject.cs | 12 ++++-------- src/runtime/runtime.cs | 12 +++--------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bee653e8..7fbd6f7b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ when .NET expects an integer [#1342][i1342] - BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. - BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. +- BREAKING: `PyObject.Length()` now raises a `PythonException` when object does not support a concept of length. - Sign Runtime DLL with a strong name - Implement loading through `clr_loader` instead of the included `ClrModule`, enables support for .NET Core diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3c7f13ec6..382ed8ccd 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -615,19 +615,15 @@ public virtual void DelItem(int index) /// - /// Length Method - /// - /// /// Returns the length for objects that support the Python sequence - /// protocol, or 0 if the object does not support the protocol. - /// + /// protocol. + /// public virtual long Length() { - var s = Runtime.PyObject_Size(obj); + var s = Runtime.PyObject_Size(Reference); if (s < 0) { - Runtime.PyErr_Clear(); - return 0; + throw new PythonException(); } return s; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec7f5e446..332222e18 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1115,13 +1115,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static int PyObject_Not(IntPtr pointer) => Delegates.PyObject_Not(pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long)_PyObject_Size(pointer); - } - - - private static IntPtr _PyObject_Size(IntPtr pointer) => Delegates._PyObject_Size(pointer); + internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); @@ -2317,7 +2311,7 @@ static Delegates() PyCallable_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCallable_Check), GetUnmanagedDll(_PythonDll)); PyObject_IsTrue = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsTrue), GetUnmanagedDll(_PythonDll)); PyObject_Not = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Not), GetUnmanagedDll(_PythonDll)); - _PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); + PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); @@ -2589,7 +2583,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyCallable_Check { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsTrue { get; } internal static delegate* unmanaged[Cdecl] PyObject_Not { get; } - internal static delegate* unmanaged[Cdecl] _PyObject_Size { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Size { get; } internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } From b2c15a4dafd6bdc02e79cf80cc48a91c2f0cb39b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:57:45 -0800 Subject: [PATCH 071/151] ImportHook: verify __import__ was not updated before overwriting it with original value --- src/runtime/importhook.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 066c765fe..0fba8f6a6 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -61,6 +61,13 @@ static void RestoreImport() { IntPtr builtins = Runtime.GetBuiltins(); + IntPtr existing = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); + Runtime.XDecref(existing); + if (existing != hook.ptr) + { + throw new NotSupportedException("Unable to restore original __import__."); + } + int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); PythonException.ThrowIfIsNotZero(res); Runtime.XDecref(py_import); @@ -374,6 +381,8 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) private static bool IsLoadAll(BorrowedReference fromList) { + if (fromList == null) throw new ArgumentNullException(nameof(fromList)); + if (CLRModule.preload) { return false; From fefa322cc49772bc5d85803a9d39fb78ce04a065 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:58:06 -0800 Subject: [PATCH 072/151] drop _PyObject_GetDictPtr and use PyObject_GenericGetDict instead --- src/runtime/importhook.cs | 8 +++++--- src/runtime/runtime.cs | 9 ++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 0fba8f6a6..14f48712a 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -95,7 +95,7 @@ internal static unsafe void Initialize() // both dicts are borrowed references BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); Runtime.PyDict_Update(mod_dict, clr_dict); BorrowedReference dict = Runtime.PyImport_GetModuleDict(); @@ -157,8 +157,10 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa // update the module dictionary with the contents of the root dictionary root.LoadNames(); BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); - Runtime.PyDict_Update(py_mod_dict, clr_dict); + using (var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference)) + { + Runtime.PyDict_Update(py_mod_dict, clr_dict); + } // find any items from the from list and get them from the root if they're not // already in the module dictionary diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 332222e18..f6a376145 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2016,9 +2016,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - - internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - + internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); @@ -2466,8 +2465,8 @@ static Delegates() PyType_Ready = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Ready), GetUnmanagedDll(_PythonDll)); _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL)); PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); - _PyObject_GetDictPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GetDictPtr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); @@ -2732,7 +2731,6 @@ static Delegates() internal static delegate* unmanaged[Cdecl] _PyType_Lookup { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } - internal static delegate* unmanaged[Cdecl] _PyObject_GetDictPtr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } @@ -2769,6 +2767,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } } } From 908316fa766b1db33bda7a28a59c81684775ac89 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 16:48:18 -0800 Subject: [PATCH 073/151] ImportHook: drop dead code around clr_prefix --- src/runtime/importhook.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 14f48712a..9caa04e60 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -259,7 +259,6 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) } string realname = mod_name; - string clr_prefix = null; // 2010-08-15: Always seemed smart to let python try first... // This shaves off a few tenths of a second on test_module.py @@ -317,10 +316,7 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) } return new NewReference(module).DangerousMoveToPointer(); } - if (clr_prefix != null) - { - return GetCLRModule(fromList).DangerousMoveToPointerOrNull(); - } + module = Runtime.PyDict_GetItemString(modules, names[0]); return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); } @@ -360,12 +356,6 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) // Add the module to sys.modules Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); - - // If imported from CLR add clr. to sys.modules as well - if (clr_prefix != null) - { - Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.ObjectReference); - } } { From db765ca714dbe1453f329189196d23bd67bc5251 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 7 Mar 2021 16:35:48 -0800 Subject: [PATCH 074/151] suggest to set Runtime.PythonDLL when embedding in .NET --- src/runtime/runtime.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a8d861770..b779c6707 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2518,7 +2518,19 @@ static Delegates() } static global::System.IntPtr GetFunctionByName(string functionName, global::System.IntPtr libraryHandle) - => libraryLoader.GetFunction(libraryHandle, functionName); + { + try + { + return libraryLoader.GetFunction(libraryHandle, functionName); + } + catch (MissingMethodException e) when (libraryHandle == IntPtr.Zero) + { + throw new MissingMethodException( + "Did you forget to set Runtime.PythonDLL?" + + " See https://github.com/pythonnet/pythonnet#embedding-python-in-net", + e); + } + } internal static delegate* unmanaged[Cdecl] PyDictProxy_New { get; } internal static delegate* unmanaged[Cdecl] Py_IncRef { get; } From 4afa4bdcb59cb664f7a69389a6d7b99289c0a192 Mon Sep 17 00:00:00 2001 From: Victor Date: Tue, 9 Mar 2021 15:52:51 -0800 Subject: [PATCH 075/151] more info on Runtime.PythonDLL --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7c2b27b93..996bfab27 100644 --- a/README.rst +++ b/README.rst @@ -45,8 +45,10 @@ module: Embedding Python in .NET ------------------------ -- You must set `Runtime.PythonDLL` or `PYTHONNET_PYDLL` environment variable +- You must set `Runtime.PythonDLL` property or `PYTHONNET_PYDLL` environment variable starting with version 3.0, otherwise you will receive `TypeInitializationException`. + Typical values are `python38.dll` (Windows), `libpython3.8.dylib` (Mac), + `libpython3.8.so` (most other *nix). - All calls to python should be inside a ``using (Py.GIL()) {/* Your code here */}`` block. - Import python modules using ``dynamic mod = Py.Import("mod")``, then From 1e5338fefb8b79ccb3268eec9ed45c2b0dc08fa1 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Tue, 23 Feb 2021 22:56:07 -0800 Subject: [PATCH 076/151] allow GIL state debugging; require GILState to be properly disposed https://github.com/pythonnet/pythonnet/issues/1389#issuecomment-784716437 --- src/runtime/pythonengine.cs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index b12309733..5925880c0 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading; namespace Python.Runtime { @@ -51,6 +52,9 @@ public static bool IsInitialized get { return initialized; } } + /// Set to true to enable GIL debugging assistance. + public static bool DebugGIL { get; set; } = false; + internal static DelegateManager DelegateManager { get @@ -633,7 +637,7 @@ public static GILState GIL() PythonEngine.Initialize(); } - return new GILState(); + return PythonEngine.DebugGIL ? new DebugGILState() : new GILState(); } public static PyScope CreateScope() @@ -658,7 +662,7 @@ internal GILState() state = PythonEngine.AcquireLock(); } - public void Dispose() + public virtual void Dispose() { if (this.isDisposed) return; @@ -669,7 +673,23 @@ public void Dispose() ~GILState() { - Dispose(); + throw new InvalidOperationException("GIL must always be released, and it must be released from the same thread that acquired it."); + } + } + + public class DebugGILState : GILState + { + readonly Thread owner; + internal DebugGILState() : base() + { + this.owner = Thread.CurrentThread; + } + public override void Dispose() + { + if (this.owner != Thread.CurrentThread) + throw new InvalidOperationException("GIL must always be released from the same thread, that acquired it"); + + base.Dispose(); } } From 483f1922fda4b477b6dfa549b65e068183c7700c Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 24 Mar 2021 22:05:02 -0700 Subject: [PATCH 077/151] try libdl.so.2 on Linux before libdl.so also refactored LibraryLoader to share code between Linux and Mac fixes https://github.com/pythonnet/pythonnet/issues/1422 --- src/runtime/platform/LibDL.cs | 109 ++++++++++++++++++++++++++ src/runtime/platform/LibraryLoader.cs | 104 +++--------------------- 2 files changed, 121 insertions(+), 92 deletions(-) create mode 100644 src/runtime/platform/LibDL.cs diff --git a/src/runtime/platform/LibDL.cs b/src/runtime/platform/LibDL.cs new file mode 100644 index 000000000..3bf2a2057 --- /dev/null +++ b/src/runtime/platform/LibDL.cs @@ -0,0 +1,109 @@ +#pragma warning disable IDE1006 // Naming Styles (interface for native functions) +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibDL + { + IntPtr dlopen(string fileName, int flags); + IntPtr dlsym(IntPtr handle, string symbol); + int dlclose(IntPtr handle); + IntPtr dlerror(); + + int RTLD_NOW { get; } + int RTLD_GLOBAL { get; } + IntPtr RTLD_DEFAULT { get; } + } + + class LinuxLibDL : ILibDL + { + private const string NativeDll = "libdl.so"; + + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x100; + public IntPtr RTLD_DEFAULT => IntPtr.Zero; + + public static ILibDL GetInstance() + { + try + { + ILibDL libdl2 = new LinuxLibDL2(); + // call dlerror to ensure library is resolved + libdl2.dlerror(); + return libdl2; + } catch (DllNotFoundException) + { + return new LinuxLibDL(); + } + } + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class LinuxLibDL2 : ILibDL + { + private const string NativeDll = "libdl.so.2"; + + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x100; + public IntPtr RTLD_DEFAULT => IntPtr.Zero; + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class MacLibDL : ILibDL + { + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x8; + const string NativeDll = "/usr/lib/libSystem.dylib"; + public IntPtr RTLD_DEFAULT => new(-2); + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } +} diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index 099b9d6cd..e361f87e4 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -28,9 +28,9 @@ public static ILibraryLoader Instance if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) _instance = new WindowsLoader(); else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - _instance = new LinuxLoader(); + _instance = new PosixLoader(LinuxLibDL.GetInstance()); else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - _instance = new DarwinLoader(); + _instance = new PosixLoader(new MacLibDL()); else throw new PlatformNotSupportedException( "This operating system is not supported" @@ -42,87 +42,19 @@ public static ILibraryLoader Instance } } - class LinuxLoader : ILibraryLoader + class PosixLoader : ILibraryLoader { - private static int RTLD_NOW = 0x2; - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; + private readonly ILibDL libDL; - public IntPtr Load(string dllToLoad) - { - ClearError(); - var res = dlopen(dllToLoad, RTLD_NOW | RTLD_GLOBAL); - if (res == IntPtr.Zero) - { - var err = GetError(); - throw new DllNotFoundException($"Could not load {dllToLoad} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); - } - - return res; - } - - public void Free(IntPtr handle) - { - dlclose(handle); - } - - public IntPtr GetFunction(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - ClearError(); - IntPtr res = dlsym(dllHandle, name); - if (res == IntPtr.Zero) - { - var err = GetError(); - throw new MissingMethodException($"Failed to load symbol {name}: {err}"); - } - return res; - } - - void ClearError() + public PosixLoader(ILibDL libDL) { - dlerror(); - } - - string GetError() - { - var res = dlerror(); - if (res != IntPtr.Zero) - return Marshal.PtrToStringAnsi(res); - else - return null; + this.libDL = libDL ?? throw new ArgumentNullException(nameof(libDL)); } - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(string fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, string symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); - } - - class DarwinLoader : ILibraryLoader - { - private static int RTLD_NOW = 0x2; - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - public IntPtr Load(string dllToLoad) { ClearError(); - var res = dlopen(dllToLoad, RTLD_NOW | RTLD_GLOBAL); + var res = libDL.dlopen(dllToLoad, libDL.RTLD_NOW | libDL.RTLD_GLOBAL); if (res == IntPtr.Zero) { var err = GetError(); @@ -134,7 +66,7 @@ public IntPtr Load(string dllToLoad) public void Free(IntPtr handle) { - dlclose(handle); + libDL.dlclose(handle); } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -142,11 +74,11 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) // look in the exe if dllHandle is NULL if (dllHandle == IntPtr.Zero) { - dllHandle = RTLD_DEFAULT; + dllHandle = libDL.RTLD_DEFAULT; } ClearError(); - IntPtr res = dlsym(dllHandle, name); + IntPtr res = libDL.dlsym(dllHandle, name); if (res == IntPtr.Zero) { var err = GetError(); @@ -157,29 +89,17 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) void ClearError() { - dlerror(); + libDL.dlerror(); } string GetError() { - var res = dlerror(); + var res = libDL.dlerror(); if (res != IntPtr.Zero) return Marshal.PtrToStringAnsi(res); else return null; } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); } class WindowsLoader : ILibraryLoader From 6e1eea6234d901f38ac028ce62863fd8fd557834 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 22:58:21 -0800 Subject: [PATCH 078/151] TypeFlags is an enum --- src/runtime/clrobject.cs | 2 +- src/runtime/interop.cs | 56 ++++++++++++++++++++------------------ src/runtime/managedtype.cs | 6 ++-- src/runtime/metatype.cs | 6 ++-- src/runtime/typemanager.cs | 16 +++++------ 5 files changed, 45 insertions(+), 41 deletions(-) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0a352b381..0aa829ee6 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,7 +14,7 @@ internal CLRObject(object ob, IntPtr tp) System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 0f56f77d9..c5958e0f7 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -344,38 +344,42 @@ public static void FreeModuleDef(IntPtr ptr) /// Note that the two values reserved for stackless have been put /// to good use as PythonNet specific flags (Managed and Subclass) /// - internal class TypeFlags + // Py_TPFLAGS_* + [Flags] + public enum TypeFlags: int { - public const int HeapType = (1 << 9); - public const int BaseType = (1 << 10); - public const int Ready = (1 << 12); - public const int Readying = (1 << 13); - public const int HaveGC = (1 << 14); + HeapType = (1 << 9), + BaseType = (1 << 10), + Ready = (1 << 12), + Readying = (1 << 13), + HaveGC = (1 << 14), // 15 and 16 are reserved for stackless - public const int HaveStacklessExtension = 0; + HaveStacklessExtension = 0, /* XXX Reusing reserved constants */ - public const int Managed = (1 << 15); // PythonNet specific - public const int Subclass = (1 << 16); // PythonNet specific - public const int HaveIndex = (1 << 17); + /// PythonNet specific + Managed = (1 << 15), + /// PythonNet specific + Subclass = (1 << 16), + HaveIndex = (1 << 17), /* Objects support nb_index in PyNumberMethods */ - public const int HaveVersionTag = (1 << 18); - public const int ValidVersionTag = (1 << 19); - public const int IsAbstract = (1 << 20); - public const int HaveNewBuffer = (1 << 21); + HaveVersionTag = (1 << 18), + ValidVersionTag = (1 << 19), + IsAbstract = (1 << 20), + HaveNewBuffer = (1 << 21), // TODO: Implement FastSubclass functions - public const int IntSubclass = (1 << 23); - public const int LongSubclass = (1 << 24); - public const int ListSubclass = (1 << 25); - public const int TupleSubclass = (1 << 26); - public const int StringSubclass = (1 << 27); - public const int UnicodeSubclass = (1 << 28); - public const int DictSubclass = (1 << 29); - public const int BaseExceptionSubclass = (1 << 30); - public const int TypeSubclass = (1 << 31); - - public const int Default = ( + IntSubclass = (1 << 23), + LongSubclass = (1 << 24), + ListSubclass = (1 << 25), + TupleSubclass = (1 << 26), + StringSubclass = (1 << 27), + UnicodeSubclass = (1 << 28), + DictSubclass = (1 << 29), + BaseExceptionSubclass = (1 << 30), + TypeSubclass = (1 << 31), + + Default = ( HaveStacklessExtension | - HaveVersionTag); + HaveVersionTag), } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 09e8a3be2..d3ee697fd 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -92,7 +92,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { IntPtr op = tp == ob @@ -117,7 +117,7 @@ internal static ManagedType GetManagedObjectType(IntPtr ob) if (ob != IntPtr.Zero) { IntPtr tp = Runtime.PyObject_TYPE(ob); - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); @@ -152,7 +152,7 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { return true; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 36b406c7b..68dae2508 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -147,13 +147,13 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - int flags = TypeFlags.Default; + var flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); @@ -285,7 +285,7 @@ public static void tp_dealloc(IntPtr tp) { // Fix this when we dont cheat on the handle for subclasses! - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e0564b243..01aceb656 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -171,9 +171,9 @@ internal static IntPtr CreateType(Type impl) SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); - int flags = TypeFlags.Default | TypeFlags.Managed | + var flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); if (Runtime.PyType_Ready(type) != 0) { @@ -286,12 +286,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - const int flags = TypeFlags.Default + const TypeFlags flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.BaseType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note @@ -457,11 +457,11 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) int size = TypeOffset.magic() + IntPtr.Size; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); - const int flags = TypeFlags.Default + const TypeFlags flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: @@ -562,11 +562,11 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); Runtime.XIncref(base_); - int flags = TypeFlags.Default; + var flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); CopySlot(base_, type, TypeOffset.tp_traverse); CopySlot(base_, type, TypeOffset.tp_clear); From 486ad0a59bf343eb460a075c9b0aa369c28310b9 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 22:58:38 -0800 Subject: [PATCH 079/151] use C# 9 everywhere by default --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index edc8ba513..e0cd93ede 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,7 +4,7 @@ Copyright (c) 2006-2020 The Contributors of the Python.NET Project pythonnet Python.NET - 7.3 + 9.0 false From 95d5e357c94a1c9bce80116436989e86477f50a0 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 23:02:52 -0800 Subject: [PATCH 080/151] added new class PyType to wrap Python type objects and enable new type construction from PyType_Spec (TypeSpec class) --- CHANGELOG.md | 1 + src/embed_tests/TestPyType.cs | 45 ++++++++++ src/runtime/TypeSpec.cs | 121 +++++++++++++++++++++++++++ src/runtime/native/NativeTypeSpec.cs | 45 ++++++++++ src/runtime/pytype.cs | 50 +++++++++++ src/runtime/runtime.cs | 4 + 6 files changed, 266 insertions(+) create mode 100644 src/embed_tests/TestPyType.cs create mode 100644 src/runtime/TypeSpec.cs create mode 100644 src/runtime/native/NativeTypeSpec.cs create mode 100644 src/runtime/pytype.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fbd6f7b4..0d9a79d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine - Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) +- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec` ### Changed - Drop support for Python 2, 3.4, and 3.5 diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs new file mode 100644 index 000000000..02142b782 --- /dev/null +++ b/src/embed_tests/TestPyType.cs @@ -0,0 +1,45 @@ +using System.Text; + +using NUnit.Framework; + +using Python.Runtime; +using Python.Runtime.Native; + +namespace Python.EmbeddingTest +{ + public class TestPyType + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void CanCreateHeapType() + { + const string name = "nÁmæ"; + const string docStr = "dÁcæ"; + + using var doc = new StrPtr(docStr, Encoding.UTF8); + var spec = new TypeSpec( + name: name, + basicSize: ObjectOffset.Size(Runtime.Runtime.PyTypeType), + slots: new TypeSpec.Slot[] { + new (TypeSlotID.tp_doc, doc.RawPointer), + }, + TypeFlags.Default | TypeFlags.HeapType + ); + + using var type = new PyType(spec); + Assert.AreEqual(name, type.GetAttr("__name__").As()); + Assert.AreEqual(docStr, type.GetAttr("__doc__").As()); + } + } +} diff --git a/src/runtime/TypeSpec.cs b/src/runtime/TypeSpec.cs new file mode 100644 index 000000000..87c0f94bc --- /dev/null +++ b/src/runtime/TypeSpec.cs @@ -0,0 +1,121 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Python.Runtime +{ + public class TypeSpec + { + public TypeSpec(string name, int basicSize, IEnumerable slots, TypeFlags flags, int itemSize = 0) + { + this.Name = name ?? throw new ArgumentNullException(nameof(name)); + this.BasicSize = basicSize; + this.Slots = slots.ToArray(); + this.Flags = flags; + this.ItemSize = itemSize; + } + public string Name { get; } + public int BasicSize { get; } + public int ItemSize { get; } + public TypeFlags Flags { get; } + public IReadOnlyList Slots { get; } + + [StructLayout(LayoutKind.Sequential)] + public struct Slot + { + public Slot(TypeSlotID id, IntPtr value) + { + ID = id; + Value = value; + } + + public TypeSlotID ID { get; } + public IntPtr Value { get; } + } + } + + public enum TypeSlotID : int + { + mp_ass_subscript = 3, + mp_length = 4, + mp_subscript = 5, + nb_absolute = 6, + nb_add = 7, + nb_and = 8, + nb_bool = 9, + nb_divmod = 10, + nb_float = 11, + nb_floor_divide = 12, + nb_index = 13, + nb_inplace_add = 14, + nb_inplace_and = 15, + nb_inplace_floor_divide = 16, + nb_inplace_lshift = 17, + nb_inplace_multiply = 18, + nb_inplace_or = 19, + nb_inplace_power = 20, + nb_inplace_remainder = 21, + nb_inplace_rshift = 22, + nb_inplace_subtract = 23, + nb_inplace_true_divide = 24, + nb_inplace_xor = 25, + nb_int = 26, + nb_invert = 27, + nb_lshift = 28, + nb_multiply = 29, + nb_negative = 30, + nb_or = 31, + nb_positive = 32, + nb_power = 33, + nb_remainder = 34, + nb_rshift = 35, + nb_subtract = 36, + nb_true_divide = 37, + nb_xor = 38, + sq_ass_item = 39, + sq_concat = 40, + sq_contains = 41, + sq_inplace_concat = 42, + sq_inplace_repeat = 43, + sq_item = 44, + sq_length = 45, + sq_repeat = 46, + tp_alloc = 47, + tp_base = 48, + tp_bases = 49, + tp_call = 50, + tp_clear = 51, + tp_dealloc = 52, + tp_del = 53, + tp_descr_get = 54, + tp_descr_set = 55, + tp_doc = 56, + tp_getattr = 57, + tp_getattro = 58, + tp_hash = 59, + tp_init = 60, + tp_is_gc = 61, + tp_iter = 62, + tp_iternext = 63, + tp_methods = 64, + tp_new = 65, + tp_repr = 66, + tp_richcompare = 67, + tp_setattr = 68, + tp_setattro = 69, + tp_str = 70, + tp_traverse = 71, + tp_members = 72, + tp_getset = 73, + tp_free = 74, + nb_matrix_multiply = 75, + nb_inplace_matrix_multiply = 76, + am_await = 77, + am_aiter = 78, + am_anext = 79, + /// New in 3.5 + tp_finalize = 80, + } +} diff --git a/src/runtime/native/NativeTypeSpec.cs b/src/runtime/native/NativeTypeSpec.cs new file mode 100644 index 000000000..d55b77381 --- /dev/null +++ b/src/runtime/native/NativeTypeSpec.cs @@ -0,0 +1,45 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime.Native +{ + [StructLayout(LayoutKind.Sequential)] + struct NativeTypeSpec : IDisposable + { + public readonly StrPtr Name; + public readonly int BasicSize; + public readonly int ItemSize; + public readonly TypeFlags Flags; + public IntPtr Slots; + + public NativeTypeSpec(TypeSpec spec) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.BasicSize = spec.BasicSize; + this.ItemSize = spec.ItemSize; + this.Flags = spec.Flags; + + unsafe + { + int slotsBytes = checked((spec.Slots.Count + 1) * Marshal.SizeOf()); + var slots = (TypeSpec.Slot*)Marshal.AllocHGlobal(slotsBytes); + for (int slotIndex = 0; slotIndex < spec.Slots.Count; slotIndex++) + slots[slotIndex] = spec.Slots[slotIndex]; + slots[spec.Slots.Count] = default; + this.Slots = (IntPtr)slots; + } + } + + public void Dispose() + { + // we have to leak the name + // this.Name.Dispose(); + Marshal.FreeHGlobal(this.Slots); + this.Slots = IntPtr.Zero; + } + } +} diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs new file mode 100644 index 000000000..8bc08b76d --- /dev/null +++ b/src/runtime/pytype.cs @@ -0,0 +1,50 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + public class PyType : PyObject + { + /// Creates heap type object from the . + public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases)) { } + /// Wraps an existing type object. + public PyType(PyObject o) : base(FromObject(o)) { } + + /// Checks if specified object is a Python type. + public static bool IsType(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return Runtime.PyType_Check(value.obj); + } + + private static BorrowedReference FromObject(PyObject o) + { + if (o is null) throw new ArgumentNullException(nameof(o)); + if (!IsType(o)) throw new ArgumentException("object is not a type"); + + return o.Reference; + } + + private static IntPtr FromSpec(TypeSpec spec, PyTuple? bases = null) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + if ((spec.Flags & TypeFlags.HeapType) == 0) + throw new NotSupportedException("Only heap types are supported"); + + var nativeSpec = new NativeTypeSpec(spec); + var basesRef = bases is null ? default : bases.Reference; + var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + + PythonException.ThrowIfIsNull(result); + + nativeSpec.Dispose(); + + return result.DangerousMoveToPointer(); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b779c6707..caa160bcf 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2006,6 +2006,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n) => Delegates.PyType_GenericAlloc(type, n); + + internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); /// /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. @@ -2509,6 +2511,7 @@ static Delegates() PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); } static global::System.IntPtr GetUnmanagedDll(string libraryName) @@ -2786,6 +2789,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } + internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } } } From 637922120306c5cb67156585cd71a65a26ffbe8d Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sat, 20 Feb 2021 14:36:04 -0800 Subject: [PATCH 081/151] reworked Enum marshaling - enums are no longer converted to and from PyLong automatically https://github.com/pythonnet/pythonnet/issues/1220 - one can construct an instance of MyEnum from Python using MyEnum(numeric_val), e.g. MyEnum(10) - in the above, if MyEnum does not have [Flags] and does not have value 10 defined, to create MyEnum with value 10 one must call MyEnum(10, True). Here True is an unnamed parameter, that allows unchecked conversion - legacy behavior has been moved to a codec (EnumPyLongCodec); enums can now be encoded by codecs - flags enums support bitwise ops via EnumOps class --- CHANGELOG.md | 3 + src/embed_tests/TestOperator.cs | 8 ++ src/runtime/Codecs/EnumPyLongCodec.cs | 68 +++++++++++++++++ src/runtime/classmanager.cs | 11 +++ src/runtime/classobject.cs | 47 ++++++++++-- src/runtime/converter.cs | 74 ++++++++---------- src/runtime/exceptions.cs | 2 +- src/runtime/operatormethod.cs | 16 ++-- src/runtime/opshelper.cs | 77 +++++++++++++++++++ src/runtime/polyfill/ReflectionPolyfills.cs | 3 + src/runtime/pylong.cs | 5 +- src/runtime/pyobject.cs | 16 ++-- src/runtime/runtime.cs | 6 +- tests/test_array.py | 2 +- tests/test_conversion.py | 35 --------- tests/test_enum.py | 84 +++++++++++---------- tests/test_indexer.py | 6 +- 17 files changed, 316 insertions(+), 147 deletions(-) create mode 100644 src/runtime/Codecs/EnumPyLongCodec.cs create mode 100644 src/runtime/opshelper.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9a79d21..e5f262620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ when .NET expects an integer [#1342][i1342] - BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. - BREAKING: `PyObject.Length()` now raises a `PythonException` when object does not support a concept of length. +- BREAKING: disabled implicit conversion from C# enums to Python `int` and back. +One must now either use enum members (e.g. `MyEnum.Option`), or use enum constructor +(e.g. `MyEnum(42)` or `MyEnum(42, True)` when `MyEnum` does not have a member with value 42). - Sign Runtime DLL with a strong name - Implement loading through `clr_loader` instead of the included `ClrModule`, enables support for .NET Core diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index 8e9feb241..68a6e8e35 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -335,6 +335,14 @@ public void SymmetricalOperatorOverloads() "); } + [Test] + public void EnumOperator() + { + PythonEngine.Exec($@" +from System.IO import FileAccess +c = FileAccess.Read | FileAccess.Write"); + } + [Test] public void OperatorOverloadMissingArgument() { diff --git a/src/runtime/Codecs/EnumPyLongCodec.cs b/src/runtime/Codecs/EnumPyLongCodec.cs new file mode 100644 index 000000000..7dab98028 --- /dev/null +++ b/src/runtime/Codecs/EnumPyLongCodec.cs @@ -0,0 +1,68 @@ +using System; + +namespace Python.Runtime.Codecs +{ + [Obsolete] + public sealed class EnumPyLongCodec : IPyObjectEncoder, IPyObjectDecoder + { + public static EnumPyLongCodec Instance { get; } = new EnumPyLongCodec(); + + public bool CanDecode(PyObject objectType, Type targetType) + { + return targetType.IsEnum + && objectType.IsSubclass(new BorrowedReference(Runtime.PyLongType)); + } + + public bool CanEncode(Type type) + { + return type == typeof(object) || type == typeof(ValueType) || type.IsEnum; + } + + public bool TryDecode(PyObject pyObj, out T value) + { + value = default; + if (!typeof(T).IsEnum) return false; + + Type etype = Enum.GetUnderlyingType(typeof(T)); + + if (!PyLong.IsLongType(pyObj)) return false; + + object result; + try + { + result = pyObj.AsManagedObject(etype); + } + catch (InvalidCastException) + { + return false; + } + + if (Enum.IsDefined(typeof(T), result) || typeof(T).IsFlagsEnum()) + { + value = (T)Enum.ToObject(typeof(T), result); + return true; + } + + return false; + } + + public PyObject TryEncode(object value) + { + if (value is null) return null; + + var enumType = value.GetType(); + if (!enumType.IsEnum) return null; + + try + { + return new PyLong((long)value); + } + catch (InvalidCastException) + { + return new PyLong((ulong)value); + } + } + + private EnumPyLongCodec() { } + } +} diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 1ee06e682..306962f56 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -403,6 +403,17 @@ private static ClassInfo GetClassInfo(Type type) } } + // only [Flags] enums support bitwise operations + if (type.IsEnum && type.IsFlagsEnum()) + { + var opsImpl = typeof(EnumOps<>).MakeGenericType(type); + foreach (var op in opsImpl.GetMethods(OpsHelper.BindingFlags)) + { + local[op.Name] = 1; + } + info = info.Concat(opsImpl.GetMethods(OpsHelper.BindingFlags)).ToArray(); + } + // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) { diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 4aa97f648..1a2532044 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -50,8 +50,9 @@ internal NewReference GetDocString() /// /// Implements __new__ for reflected classes and value types. /// - public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) + public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) { + var tp = new BorrowedReference(tpRaw); var self = GetManagedObject(tp) as ClassObject; // Sanity check: this ensures a graceful error if someone does @@ -87,7 +88,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - return CLRObject.GetInstHandle(result, tp); + return CLRObject.GetInstHandle(result, tp).DangerousMoveToPointerOrNull(); } if (type.IsAbstract) @@ -98,8 +99,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) if (type.IsEnum) { - Exceptions.SetError(Exceptions.TypeError, "cannot instantiate enumeration"); - return IntPtr.Zero; + return NewEnum(type, new BorrowedReference(args), tp).DangerousMoveToPointerOrNull(); } object obj = self.binder.InvokeRaw(IntPtr.Zero, args, kw); @@ -108,7 +108,44 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - return CLRObject.GetInstHandle(obj, tp); + return CLRObject.GetInstHandle(obj, tp).DangerousMoveToPointerOrNull(); + } + + private static NewReference NewEnum(Type type, BorrowedReference args, BorrowedReference tp) + { + nint argCount = Runtime.PyTuple_Size(args); + bool allowUnchecked = false; + if (argCount == 2) + { + var allow = Runtime.PyTuple_GetItem(args, 1); + if (!Converter.ToManaged(allow, typeof(bool), out var allowObj, true) || allowObj is null) + { + Exceptions.RaiseTypeError("second argument to enum constructor must be a boolean"); + return default; + } + allowUnchecked |= (bool)allowObj; + } + + if (argCount < 1 || argCount > 2) + { + Exceptions.SetError(Exceptions.TypeError, "no constructors match given arguments"); + return default; + } + + var op = Runtime.PyTuple_GetItem(args, 0); + if (!Converter.ToManaged(op, type.GetEnumUnderlyingType(), out object result, true)) + { + return default; + } + + if (!allowUnchecked && !Enum.IsDefined(type, result) && !type.IsFlagsEnum()) + { + Exceptions.SetError(Exceptions.ValueError, "Invalid enumeration value. Pass True as the second argument if unchecked conversion is desired"); + return default; + } + + object enumValue = Enum.ToObject(type, result); + return CLRObject.GetInstHandle(enumValue, tp); } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index f3b378113..4de334b5f 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -27,7 +27,6 @@ private Converter() private static Type int16Type; private static Type int32Type; private static Type int64Type; - private static Type flagsType; private static Type boolType; private static Type typeType; @@ -42,7 +41,6 @@ static Converter() singleType = typeof(Single); doubleType = typeof(Double); decimalType = typeof(Decimal); - flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); } @@ -148,7 +146,8 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object) + || type.IsEnum) { var encoded = PyObjectConversions.TryEncode(value, type); if (encoded != null) { result = encoded.Handle; @@ -203,6 +202,11 @@ internal static IntPtr ToPython(object value, Type type) type = value.GetType(); + if (type.IsEnum) + { + return CLRObject.GetInstHandle(value, type); + } + TypeCode tc = Type.GetTypeCode(type); switch (tc) @@ -317,6 +321,18 @@ internal static bool ToManaged(IntPtr value, Type type, } return Converter.ToManagedValue(value, type, out result, setError); } + /// + /// Return a managed object for the given Python object, taking funny + /// byref types into account. + /// + /// A Python object + /// The desired managed type + /// Receives the managed object + /// If true, call Exceptions.SetError with the reason for failure. + /// True on success + internal static bool ToManaged(BorrowedReference value, Type type, + out object result, bool setError) + => ToManaged(value.DangerousGetAddress(), type, out result, setError); internal static bool ToManagedValue(BorrowedReference value, Type obType, out object result, bool setError) @@ -398,11 +414,6 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToArray(value, obType, out result, setError); } - if (obType.IsEnum) - { - return ToEnum(value, obType, out result, setError); - } - // Conversion to 'Object' is done based on some reasonable default // conversions (Python string -> managed string, Python int -> Int32 etc.). if (obType == objectType) @@ -497,7 +508,7 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } TypeCode typeCode = Type.GetTypeCode(obType); - if (typeCode == TypeCode.Object) + if (typeCode == TypeCode.Object || obType.IsEnum) { IntPtr pyType = Runtime.PyObject_TYPE(value); if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) @@ -516,8 +527,17 @@ internal static bool ToManagedValue(IntPtr value, Type obType, /// private static bool ToPrimitive(IntPtr value, Type obType, out object result, bool setError) { - TypeCode tc = Type.GetTypeCode(obType); result = null; + if (obType.IsEnum) + { + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, "since Python.NET 3.0 int can not be converted to Enum implicitly. Use Enum(int_value)"); + } + return false; + } + + TypeCode tc = Type.GetTypeCode(obType); IntPtr op = IntPtr.Zero; switch (tc) @@ -876,40 +896,6 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s result = items; return true; } - - - /// - /// Convert a Python value to a correctly typed managed enum instance. - /// - private static bool ToEnum(IntPtr value, Type obType, out object result, bool setError) - { - Type etype = Enum.GetUnderlyingType(obType); - result = null; - - if (!ToPrimitive(value, etype, out result, setError)) - { - return false; - } - - if (Enum.IsDefined(obType, result)) - { - result = Enum.ToObject(obType, result); - return true; - } - - if (obType.GetCustomAttributes(flagsType, true).Length > 0) - { - result = Enum.ToObject(obType, result); - return true; - } - - if (setError) - { - Exceptions.SetError(Exceptions.ValueError, "invalid enumeration value"); - } - - return false; - } } public static class ConverterExtension diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index da8653853..06d2d55b5 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -340,7 +340,7 @@ public static void Clear() public static void warn(string message, IntPtr exception, int stacklevel) { if (exception == IntPtr.Zero || - (Runtime.PyObject_IsSubclass(exception, Exceptions.Warning) != 1)) + (Runtime.PyObject_IsSubclass(new BorrowedReference(exception), new BorrowedReference(Exceptions.Warning)) != 1)) { Exceptions.RaiseTypeError("Invalid exception"); } diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs index 59bf944bc..e44dc3be1 100644 --- a/src/runtime/operatormethod.cs +++ b/src/runtime/operatormethod.cs @@ -51,7 +51,6 @@ static OperatorMethod() ["op_OnesComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), ["op_UnaryNegation"] = new SlotDefinition("__neg__", TypeOffset.nb_negative), ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), - ["op_OneComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), }; ComparisonOpMap = new Dictionary { @@ -80,7 +79,7 @@ public static void Shutdown() public static bool IsOperatorMethod(MethodBase method) { - if (!method.IsSpecialName) + if (!method.IsSpecialName && !method.IsOpsHelper()) { return false; } @@ -102,7 +101,12 @@ public static void FixupSlots(IntPtr pyType, Type clrType) { const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; Debug.Assert(_opType != null); - foreach (var method in clrType.GetMethods(flags)) + + var staticMethods = + clrType.IsEnum ? typeof(EnumOps<>).MakeGenericType(clrType).GetMethods(flags) + : clrType.GetMethods(flags); + + foreach (var method in staticMethods) { // We only want to override slots for operators excluding // comparison operators, which are handled by ClassBase.tp_richcompare. @@ -170,9 +174,11 @@ public static string ReversePyMethodName(string pyName) /// public static bool IsReverse(MethodInfo method) { - Type declaringType = method.DeclaringType; + Type primaryType = method.IsOpsHelper() + ? method.DeclaringType.GetGenericArguments()[0] + : method.DeclaringType; Type leftOperandType = method.GetParameters()[0].ParameterType; - return leftOperandType != declaringType; + return leftOperandType != primaryType; } public static void FilterMethods(MethodInfo[] methods, out MethodInfo[] forwardMethods, out MethodInfo[] reverseMethods) diff --git a/src/runtime/opshelper.cs b/src/runtime/opshelper.cs new file mode 100644 index 000000000..59f7704b7 --- /dev/null +++ b/src/runtime/opshelper.cs @@ -0,0 +1,77 @@ +using System; +using System.Linq.Expressions; +using System.Reflection; + +using static Python.Runtime.OpsHelper; + +namespace Python.Runtime +{ + static class OpsHelper + { + public static BindingFlags BindingFlags => BindingFlags.Public | BindingFlags.Static; + + public static Func Binary(Func func) + { + var a = Expression.Parameter(typeof(T), "a"); + var b = Expression.Parameter(typeof(T), "b"); + var body = func(a, b); + var lambda = Expression.Lambda>(body, a, b); + return lambda.Compile(); + } + + public static Func Unary(Func func) + { + var value = Expression.Parameter(typeof(T), "value"); + var body = func(value); + var lambda = Expression.Lambda>(body, value); + return lambda.Compile(); + } + + public static bool IsOpsHelper(this MethodBase method) + => method.DeclaringType.GetCustomAttribute() is not null; + + public static Expression EnumUnderlyingValue(Expression enumValue) + => Expression.Convert(enumValue, enumValue.Type.GetEnumUnderlyingType()); + } + + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + internal class OpsAttribute: Attribute { } + + [Ops] + internal static class EnumOps where T : Enum + { + static readonly Func and = BinaryOp(Expression.And); + static readonly Func or = BinaryOp(Expression.Or); + static readonly Func xor = BinaryOp(Expression.ExclusiveOr); + + static readonly Func invert = UnaryOp(Expression.OnesComplement); + + public static T op_BitwiseAnd(T a, T b) => and(a, b); + public static T op_BitwiseOr(T a, T b) => or(a, b); + public static T op_ExclusiveOr(T a, T b) => xor(a, b); + public static T op_OnesComplement(T value) => invert(value); + + static Expression FromNumber(Expression number) + => Expression.Convert(number, typeof(T)); + + static Func BinaryOp(Func op) + { + return Binary((a, b) => + { + var numericA = EnumUnderlyingValue(a); + var numericB = EnumUnderlyingValue(b); + var numericResult = op(numericA, numericB); + return FromNumber(numericResult); + }); + } + static Func UnaryOp(Func op) + { + return Unary(value => + { + var numeric = EnumUnderlyingValue(value); + var numericResult = op(numeric); + return FromNumber(numericResult); + }); + } + } +} diff --git a/src/runtime/polyfill/ReflectionPolyfills.cs b/src/runtime/polyfill/ReflectionPolyfills.cs index 65f9b83de..36bd39cef 100644 --- a/src/runtime/polyfill/ReflectionPolyfills.cs +++ b/src/runtime/polyfill/ReflectionPolyfills.cs @@ -30,5 +30,8 @@ public static T GetCustomAttribute(this Assembly assembly) where T: Attribute .Cast() .SingleOrDefault(); } + + public static bool IsFlagsEnum(this Type type) + => type.GetCustomAttribute() is not null; } } diff --git a/src/runtime/pylong.cs b/src/runtime/pylong.cs index fdfd26aba..8cb814cf6 100644 --- a/src/runtime/pylong.cs +++ b/src/runtime/pylong.cs @@ -188,11 +188,8 @@ public PyLong(string value) : base(FromString(value)) /// - /// IsLongType Method - /// - /// /// Returns true if the given object is a Python long. - /// + /// public static bool IsLongType(PyObject value) { return Runtime.PyLong_Check(value.obj); diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 382ed8ccd..81578a7a8 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -930,17 +930,21 @@ public bool IsInstance(PyObject typeOrClass) /// - /// IsSubclass Method - /// - /// - /// Return true if the object is identical to or derived from the + /// Return true if the object is identical to or derived from the /// given Python type or class. This method always succeeds. - /// + /// public bool IsSubclass(PyObject typeOrClass) { if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); - int r = Runtime.PyObject_IsSubclass(obj, typeOrClass.obj); + return IsSubclass(typeOrClass.Reference); + } + + internal bool IsSubclass(BorrowedReference typeOrClass) + { + if (typeOrClass.IsNull) throw new ArgumentNullException(nameof(typeOrClass)); + + int r = Runtime.PyObject_IsSubclass(Reference, typeOrClass); if (r < 0) { Runtime.PyErr_Clear(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index caa160bcf..263b4473e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1109,7 +1109,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static int PyObject_IsInstance(IntPtr ob, IntPtr type) => Delegates.PyObject_IsInstance(ob, type); - internal static int PyObject_IsSubclass(IntPtr ob, IntPtr type) => Delegates.PyObject_IsSubclass(ob, type); + internal static int PyObject_IsSubclass(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsSubclass(ob, type); internal static int PyCallable_Check(IntPtr pointer) => Delegates.PyCallable_Check(pointer); @@ -2314,7 +2314,7 @@ static Delegates() PyObject_CallObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_CallObject), GetUnmanagedDll(_PythonDll)); PyObject_RichCompareBool = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_RichCompareBool), GetUnmanagedDll(_PythonDll)); PyObject_IsInstance = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsInstance), GetUnmanagedDll(_PythonDll)); - PyObject_IsSubclass = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsSubclass), GetUnmanagedDll(_PythonDll)); + PyObject_IsSubclass = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsSubclass), GetUnmanagedDll(_PythonDll)); PyCallable_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCallable_Check), GetUnmanagedDll(_PythonDll)); PyObject_IsTrue = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsTrue), GetUnmanagedDll(_PythonDll)); PyObject_Not = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Not), GetUnmanagedDll(_PythonDll)); @@ -2599,7 +2599,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_CallObject { get; } internal static delegate* unmanaged[Cdecl] PyObject_RichCompareBool { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsInstance { get; } - internal static delegate* unmanaged[Cdecl] PyObject_IsSubclass { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsSubclass { get; } internal static delegate* unmanaged[Cdecl] PyCallable_Check { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsTrue { get; } internal static delegate* unmanaged[Cdecl] PyObject_Not { get; } diff --git a/tests/test_array.py b/tests/test_array.py index 2b1a289ad..d6f08a961 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -680,7 +680,7 @@ def test_enum_array(): items[-1] = ShortEnum.Zero assert items[-1] == ShortEnum.Zero - with pytest.raises(ValueError): + with pytest.raises(TypeError): ob = Test.EnumArrayTest() ob.items[0] = 99 diff --git a/tests/test_conversion.py b/tests/test_conversion.py index aea95e164..eec2bcde6 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -601,41 +601,6 @@ class Foo(object): assert ob.ObjectField == Foo -def test_enum_conversion(): - """Test enum conversion.""" - from Python.Test import ShortEnum - - ob = ConversionTest() - assert ob.EnumField == ShortEnum.Zero - - ob.EnumField = ShortEnum.One - assert ob.EnumField == ShortEnum.One - - ob.EnumField = 0 - assert ob.EnumField == ShortEnum.Zero - assert ob.EnumField == 0 - - ob.EnumField = 1 - assert ob.EnumField == ShortEnum.One - assert ob.EnumField == 1 - - with pytest.raises(ValueError): - ob = ConversionTest() - ob.EnumField = 10 - - with pytest.raises(ValueError): - ob = ConversionTest() - ob.EnumField = 255 - - with pytest.raises(OverflowError): - ob = ConversionTest() - ob.EnumField = 1000000 - - with pytest.raises(TypeError): - ob = ConversionTest() - ob.EnumField = "spam" - - def test_null_conversion(): """Test null conversion.""" import System diff --git a/tests/test_enum.py b/tests/test_enum.py index 27fe7e9ef..1f0711a94 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -22,69 +22,69 @@ def test_enum_get_member(): """Test access to enum members.""" from System import DayOfWeek - assert DayOfWeek.Sunday == 0 - assert DayOfWeek.Monday == 1 - assert DayOfWeek.Tuesday == 2 - assert DayOfWeek.Wednesday == 3 - assert DayOfWeek.Thursday == 4 - assert DayOfWeek.Friday == 5 - assert DayOfWeek.Saturday == 6 + assert DayOfWeek.Sunday == DayOfWeek(0) + assert DayOfWeek.Monday == DayOfWeek(1) + assert DayOfWeek.Tuesday == DayOfWeek(2) + assert DayOfWeek.Wednesday == DayOfWeek(3) + assert DayOfWeek.Thursday == DayOfWeek(4) + assert DayOfWeek.Friday == DayOfWeek(5) + assert DayOfWeek.Saturday == DayOfWeek(6) def test_byte_enum(): """Test byte enum.""" - assert Test.ByteEnum.Zero == 0 - assert Test.ByteEnum.One == 1 - assert Test.ByteEnum.Two == 2 + assert Test.ByteEnum.Zero == Test.ByteEnum(0) + assert Test.ByteEnum.One == Test.ByteEnum(1) + assert Test.ByteEnum.Two == Test.ByteEnum(2) def test_sbyte_enum(): """Test sbyte enum.""" - assert Test.SByteEnum.Zero == 0 - assert Test.SByteEnum.One == 1 - assert Test.SByteEnum.Two == 2 + assert Test.SByteEnum.Zero == Test.SByteEnum(0) + assert Test.SByteEnum.One == Test.SByteEnum(1) + assert Test.SByteEnum.Two == Test.SByteEnum(2) def test_short_enum(): """Test short enum.""" - assert Test.ShortEnum.Zero == 0 - assert Test.ShortEnum.One == 1 - assert Test.ShortEnum.Two == 2 + assert Test.ShortEnum.Zero == Test.ShortEnum(0) + assert Test.ShortEnum.One == Test.ShortEnum(1) + assert Test.ShortEnum.Two == Test.ShortEnum(2) def test_ushort_enum(): """Test ushort enum.""" - assert Test.UShortEnum.Zero == 0 - assert Test.UShortEnum.One == 1 - assert Test.UShortEnum.Two == 2 + assert Test.UShortEnum.Zero == Test.UShortEnum(0) + assert Test.UShortEnum.One == Test.UShortEnum(1) + assert Test.UShortEnum.Two == Test.UShortEnum(2) def test_int_enum(): """Test int enum.""" - assert Test.IntEnum.Zero == 0 - assert Test.IntEnum.One == 1 - assert Test.IntEnum.Two == 2 + assert Test.IntEnum.Zero == Test.IntEnum(0) + assert Test.IntEnum.One == Test.IntEnum(1) + assert Test.IntEnum.Two == Test.IntEnum(2) def test_uint_enum(): """Test uint enum.""" - assert Test.UIntEnum.Zero == 0 - assert Test.UIntEnum.One == 1 - assert Test.UIntEnum.Two == 2 + assert Test.UIntEnum.Zero == Test.UIntEnum(0) + assert Test.UIntEnum.One == Test.UIntEnum(1) + assert Test.UIntEnum.Two == Test.UIntEnum(2) def test_long_enum(): """Test long enum.""" - assert Test.LongEnum.Zero == 0 - assert Test.LongEnum.One == 1 - assert Test.LongEnum.Two == 2 + assert Test.LongEnum.Zero == Test.LongEnum(0) + assert Test.LongEnum.One == Test.LongEnum(1) + assert Test.LongEnum.Two == Test.LongEnum(2) def test_ulong_enum(): """Test ulong enum.""" - assert Test.ULongEnum.Zero == 0 - assert Test.ULongEnum.One == 1 - assert Test.ULongEnum.Two == 2 + assert Test.ULongEnum.Zero == Test.ULongEnum(0) + assert Test.ULongEnum.One == Test.ULongEnum(1) + assert Test.ULongEnum.Two == Test.ULongEnum(2) def test_instantiate_enum_fails(): @@ -117,29 +117,31 @@ def test_enum_set_member_fails(): del DayOfWeek.Sunday -def test_enum_with_flags_attr_conversion(): +def test_enum_undefined_value(): """Test enumeration conversion with FlagsAttribute set.""" # This works because the FlagsField enum has FlagsAttribute. - Test.FieldTest().FlagsField = 99 + Test.FieldTest().FlagsField = Test.FlagsEnum(99) # This should fail because our test enum doesn't have it. with pytest.raises(ValueError): - Test.FieldTest().EnumField = 99 - + Test.FieldTest().EnumField = Test.ShortEnum(20) + + # explicitly permit undefined values + Test.FieldTest().EnumField = Test.ShortEnum(20, True) def test_enum_conversion(): """Test enumeration conversion.""" ob = Test.FieldTest() - assert ob.EnumField == 0 + assert ob.EnumField == Test.ShortEnum(0) ob.EnumField = Test.ShortEnum.One - assert ob.EnumField == 1 - - with pytest.raises(ValueError): - Test.FieldTest().EnumField = 20 + assert ob.EnumField == Test.ShortEnum(1) with pytest.raises(OverflowError): - Test.FieldTest().EnumField = 100000 + Test.FieldTest().EnumField = Test.ShortEnum(100000) with pytest.raises(TypeError): Test.FieldTest().EnumField = "str" + + with pytest.raises(TypeError): + Test.FieldTest().EnumField = 1 diff --git a/tests/test_indexer.py b/tests/test_indexer.py index 7992f76b0..0af6e6c45 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -400,8 +400,10 @@ def test_enum_indexer(): ob[key] = "eggs" assert ob[key] == "eggs" - ob[1] = "spam" - assert ob[1] == "spam" + with pytest.raises(TypeError): + ob[1] = "spam" + with pytest.raises(TypeError): + ob[1] with pytest.raises(TypeError): ob = Test.EnumIndexerTest() From a157fb4aa8ac9ad6955c8613e63456edeedeea8c Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Tue, 30 Mar 2021 14:45:04 -0700 Subject: [PATCH 082/151] added regression test for https://github.com/pythonnet/pythonnet/issues/1427 (stack overflow trying to pass `System.Type` to `CanEncode`) --- src/embed_tests/Codecs.cs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 266badb9e..ce837d481 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -296,6 +296,31 @@ public void IterableDecoderTest() Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); } + + // regression for https://github.com/pythonnet/pythonnet/issues/1427 + [Test] + public void PythonRegisteredDecoder_NoStackOverflowOnSystemType() + { + const string PyCode = @" +import clr +import System +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder + + +class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = 'Dummy' + def CanEncode(self, clr_type): + return clr_type.Name == 'IList`1' and clr_type.Namespace == 'System.Collections.Generic' + + +list_encoder = ListAsRawEncoder() +PyObjectConversions.RegisterEncoder(list_encoder) + +system_type = list_encoder.GetType()"; + + PythonEngine.Exec(PyCode); + } } /// From 957a3cba82950322adcd527db0a85a2de8a7de12 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Tue, 30 Mar 2021 15:03:40 -0700 Subject: [PATCH 083/151] disabled ability to register encoders for `System.Type` Without this restriction encoders created in Python cause stack overflow due to repeated attempts to pass `System.Type` instance to `CanDecode`, which requires encoding the instance of `System.Type`, et. cetera fixes https://github.com/pythonnet/pythonnet/issues/1427 --- CHANGELOG.md | 1 + src/runtime/converter.cs | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5f262620..b565fbbdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - Sign Runtime DLL with a strong name - Implement loading through `clr_loader` instead of the included `ClrModule`, enables support for .NET Core +- BREAKING: custom encoders are no longer called for instances of `System.Type` ### Fixed diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4de334b5f..cd9477a62 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -146,8 +146,11 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object) - || type.IsEnum) { + if (Type.GetTypeCode(type) == TypeCode.Object + && value.GetType() != typeof(object) + && value is not Type + || type.IsEnum + ) { var encoded = PyObjectConversions.TryEncode(value, type); if (encoded != null) { result = encoded.Handle; From 23527d11374c4149a279de540a6bc27cd8d4c38f Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Tue, 30 Mar 2021 20:59:53 -0700 Subject: [PATCH 084/151] disabled PythonRegisteredDecoder_NoStackOverflowOnSystemType test due to known issue with custom codes + engine restarts https://github.com/pythonnet/pythonnet/issues/1256 --- src/embed_tests/Codecs.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index ce837d481..de5882b32 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -298,6 +298,7 @@ public void IterableDecoderTest() } // regression for https://github.com/pythonnet/pythonnet/issues/1427 + [Ignore("https://github.com/pythonnet/pythonnet/issues/1256")] [Test] public void PythonRegisteredDecoder_NoStackOverflowOnSystemType() { From c0fe430e5a32851a3133f88c2462d9362cfe2055 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 8 Apr 2021 23:32:25 -0700 Subject: [PATCH 085/151] reworked `PythonException`: Removed private fields, apart from ones returned by `PyErr_Fetch`. Corresponding property values are now generated on demand. Added FetchCurrent*Raw for internal consumption. `PythonException.Type` is now of type `PyType`. Use C API functions `PyException_GetCause` and `PyException_GetTraceback` instead of trying to read via attributes by name. `PythonException` instances are no longer disposable. You can still dispose `.Type`, `.Value` and `.Traceback`, but it is not recommended, as they may be shared with other instances. --- CHANGELOG.md | 2 + src/embed_tests/Codecs.cs | 44 ++-- src/embed_tests/TestCallbacks.cs | 2 +- src/embed_tests/TestPyFloat.cs | 4 +- src/embed_tests/TestPyInt.cs | 4 +- src/embed_tests/TestPyList.cs | 2 +- src/embed_tests/TestPyLong.cs | 4 +- src/embed_tests/TestPyTuple.cs | 4 +- src/embed_tests/TestPyType.cs | 1 + src/embed_tests/TestPyWith.cs | 2 +- src/embed_tests/TestPythonException.cs | 64 +++-- src/embed_tests/pyimport.cs | 3 +- src/embed_tests/pyinitialize.cs | 2 +- src/runtime/StolenReference.cs | 34 ++- src/runtime/classmanager.cs | 2 +- src/runtime/converter.cs | 3 + src/runtime/delegatemanager.cs | 10 +- src/runtime/exceptions.cs | 67 +++-- src/runtime/finalizer.cs | 2 +- src/runtime/importhook.cs | 4 +- src/runtime/methodbinder.cs | 2 +- src/runtime/moduleobject.cs | 4 +- src/runtime/pybuffer.cs | 8 +- src/runtime/pyiter.cs | 2 +- src/runtime/pyobject.cs | 28 ++- src/runtime/pythonengine.cs | 18 +- src/runtime/pythonexception.cs | 324 ++++++++++++------------- src/runtime/pytuple.cs | 2 +- src/runtime/pytype.cs | 34 +++ src/runtime/runtime.cs | 55 +++-- src/runtime/typemanager.cs | 12 +- 31 files changed, 415 insertions(+), 334 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3081e2e84..5871e7ffb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru support for .NET Core - .NET and Python exceptions are preserved when crossing Python/.NET boundary - BREAKING: custom encoders are no longer called for instances of `System.Type` +- `PythonException.Restore` no longer clears `PythonException` instance. ### Fixed @@ -74,6 +75,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru ### Removed - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) +- messages in `PythonException` no longer start with exception type - support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 (see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index c36864e69..f0c00a6d8 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -325,56 +325,58 @@ def CanEncode(self, clr_type): const string TestExceptionMessage = "Hello World!"; [Test] - public void ExceptionEncoded() { + public void ExceptionEncoded() + { PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); var callMeAction = new Action(CallMe); - using (var _ = Py.GIL()) - using (var scope = Py.CreateScope()) - { - scope.Exec(@" + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + scope.Exec(@" def call(func): try: func() except ValueError as e: return str(e) "); - var callFunc = scope.Get("call"); - string message = callFunc.Invoke(callMeAction.ToPython()).As(); - Assert.AreEqual(TestExceptionMessage, message); - } + var callFunc = scope.Get("call"); + string message = callFunc.Invoke(callMeAction.ToPython()).As(); + Assert.AreEqual(TestExceptionMessage, message); } [Test] - public void ExceptionDecoded() { + public void ExceptionDecoded() + { PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); - using (var _ = Py.GIL()) - using (var scope = Py.CreateScope()) - { - var error = Assert.Throws(() - => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); - Assert.AreEqual(TestExceptionMessage, error.Message); - } + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() + => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); + Assert.AreEqual(TestExceptionMessage, error.Message); } - class ValueErrorWrapper : Exception { + class ValueErrorWrapper : Exception + { public ValueErrorWrapper(string message) : base(message) { } } - class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder { + class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder + { public bool CanDecode(PyObject objectType, Type targetType) => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError")); public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) || typeof(ValueErrorWrapper).IsSubclassOf(type); - public bool TryDecode(PyObject pyObj, out T value) { + public bool TryDecode(PyObject pyObj, out T value) + { var message = pyObj.GetAttr("args")[0].As(); value = (T)(object)new ValueErrorWrapper(message); return true; } - public PyObject TryEncode(object value) { + public PyObject TryEncode(object value) + { var error = (ValueErrorWrapper)value; return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); } diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs index 454c97578..6875fde01 100644 --- a/src/embed_tests/TestCallbacks.cs +++ b/src/embed_tests/TestCallbacks.cs @@ -24,7 +24,7 @@ public void TestNoOverloadException() { using (Py.GIL()) { dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); - Assert.AreEqual("TypeError", error.PythonTypeName); + Assert.AreEqual("TypeError", error.Type.Name); string expectedArgTypes = "()"; StringAssert.EndsWith(expectedArgTypes, error.Message); } diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index 94e7026c7..906c8cb0d 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -95,7 +95,7 @@ public void StringBadCtor() var ex = Assert.Throws(() => a = new PyFloat(i)); - StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message); + StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } @@ -132,7 +132,7 @@ public void AsFloatBad() PyFloat a = null; var ex = Assert.Throws(() => a = PyFloat.AsFloat(s)); - StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message); + StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 005ab466d..bd6cf23a1 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -128,7 +128,7 @@ public void TestCtorBadString() var ex = Assert.Throws(() => a = new PyInt(i)); - StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message); + StringAssert.StartsWith("invalid literal for int", ex.Message); Assert.IsNull(a); } @@ -161,7 +161,7 @@ public void TestAsIntBad() PyInt a = null; var ex = Assert.Throws(() => a = PyInt.AsInt(s)); - StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message); + StringAssert.StartsWith("invalid literal for int", ex.Message); Assert.IsNull(a); } diff --git a/src/embed_tests/TestPyList.cs b/src/embed_tests/TestPyList.cs index e9acfbb45..eee129f2d 100644 --- a/src/embed_tests/TestPyList.cs +++ b/src/embed_tests/TestPyList.cs @@ -41,7 +41,7 @@ public void TestStringAsListType() var ex = Assert.Throws(() => t = PyList.AsList(i)); - Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message); + Assert.AreEqual("'int' object is not iterable", ex.Message); Assert.IsNull(t); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index 3c155f315..6d587d064 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -144,7 +144,7 @@ public void TestCtorBadString() var ex = Assert.Throws(() => a = new PyLong(i)); - StringAssert.StartsWith("ValueError : invalid literal", ex.Message); + StringAssert.StartsWith("invalid literal", ex.Message); Assert.IsNull(a); } @@ -177,7 +177,7 @@ public void TestAsLongBad() PyLong a = null; var ex = Assert.Throws(() => a = PyLong.AsLong(s)); - StringAssert.StartsWith("ValueError : invalid literal", ex.Message); + StringAssert.StartsWith("invalid literal", ex.Message); Assert.IsNull(a); } diff --git a/src/embed_tests/TestPyTuple.cs b/src/embed_tests/TestPyTuple.cs index 362251049..5d76116aa 100644 --- a/src/embed_tests/TestPyTuple.cs +++ b/src/embed_tests/TestPyTuple.cs @@ -104,7 +104,7 @@ public void TestPyTupleInvalidAppend() var ex = Assert.Throws(() => t.Concat(s)); - StringAssert.StartsWith("TypeError : can only concatenate tuple", ex.Message); + StringAssert.StartsWith("can only concatenate tuple", ex.Message); Assert.AreEqual(0, t.Length()); Assert.IsEmpty(t); } @@ -164,7 +164,7 @@ public void TestInvalidAsTuple() var ex = Assert.Throws(() => t = PyTuple.AsTuple(i)); - Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message); + Assert.AreEqual("'int' object is not iterable", ex.Message); Assert.IsNull(t); } } diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 02142b782..d3937b064 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -39,6 +39,7 @@ public void CanCreateHeapType() using var type = new PyType(spec); Assert.AreEqual(name, type.GetAttr("__name__").As()); + Assert.AreEqual(name, type.Name); Assert.AreEqual(docStr, type.GetAttr("__doc__").As()); } } diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index dcd539504..c6228f1b9 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -51,7 +51,7 @@ def fail(self): catch (PythonException e) { TestContext.Out.WriteLine(e.Message); - Assert.IsTrue(e.Message.Contains("ZeroDivisionError")); + Assert.IsTrue(e.Type.Name == "ZeroDivisionError"); } } diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 702b6c3b1..0763bfb34 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -30,29 +30,29 @@ public void TestMessage() var ex = Assert.Throws(() => foo = list[0]); - Assert.AreEqual("IndexError : list index out of range", ex.Message); + Assert.AreEqual("list index out of range", ex.Message); Assert.IsNull(foo); } [Test] - public void TestNoError() + public void TestType() { - var e = new PythonException(); // There is no PyErr to fetch - Assert.AreEqual("", e.Message); + var list = new PyList(); + PyObject foo = null; + + var ex = Assert.Throws(() => foo = list[0]); + + Assert.AreEqual("IndexError", ex.Type.Name); + Assert.IsNull(foo); } [Test] - public void TestPythonErrorTypeName() + public void TestNoError() { - try - { - var module = PyModule.Import("really____unknown___module"); - Assert.Fail("Unknown module should not be loaded"); - } - catch (PythonException ex) - { - Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError")); - } + // There is no PyErr to fetch + Assert.Throws(() => PythonException.FetchCurrentRaw()); + var currentError = PythonException.FetchCurrentOrNullRaw(); + Assert.IsNull(currentError); } [Test] @@ -70,7 +70,7 @@ raise Exception('outer') from ex catch (PythonException ex) { Assert.That(ex.InnerException, Is.InstanceOf()); - Assert.That(ex.InnerException.Message, Is.EqualTo("Exception : inner")); + Assert.That(ex.InnerException.Message, Is.EqualTo("inner")); } } @@ -113,13 +113,6 @@ public void TestPythonExceptionFormat() } } - [Test] - public void TestPythonExceptionFormatNoError() - { - var ex = new PythonException(); - Assert.AreEqual(ex.StackTrace, ex.Format()); - } - [Test] public void TestPythonExceptionFormatNoTraceback() { @@ -162,22 +155,19 @@ def __init__(self, val): Assert.IsTrue(scope.TryGet("TestException", out PyObject type)); PyObject str = "dummy string".ToPython(); - IntPtr typePtr = type.Handle; - IntPtr strPtr = str.Handle; - IntPtr tbPtr = Runtime.Runtime.None.Handle; - Runtime.Runtime.XIncref(typePtr); - Runtime.Runtime.XIncref(strPtr); - Runtime.Runtime.XIncref(tbPtr); + var typePtr = new NewReference(type.Reference); + var strPtr = new NewReference(str.Reference); + var tbPtr = new NewReference(Runtime.Runtime.None.Reference); Runtime.Runtime.PyErr_NormalizeException(ref typePtr, ref strPtr, ref tbPtr); - using (PyObject typeObj = new PyObject(typePtr), strObj = new PyObject(strPtr), tbObj = new PyObject(tbPtr)) - { - // the type returned from PyErr_NormalizeException should not be the same type since a new - // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typePtr); - // the message should now be the string from the throw exception during normalization - Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); - } + using var typeObj = typePtr.MoveToPyObject(); + using var strObj = strPtr.MoveToPyObject(); + using var tbObj = tbPtr.MoveToPyObject(); + // the type returned from PyErr_NormalizeException should not be the same type since a new + // exception was raised by initializing the exception + Assert.AreNotEqual(type.Handle, typeObj.Handle); + // the message should now be the string from the throw exception during normalization + Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } } @@ -185,7 +175,7 @@ def __init__(self, val): public void TestPythonException_Normalize_ThrowsWhenErrorSet() { Exceptions.SetError(Exceptions.TypeError, "Error!"); - var pythonException = new PythonException(); + var pythonException = PythonException.FetchCurrentRaw(); Exceptions.SetError(Exceptions.TypeError, "Another error"); Assert.Throws(() => pythonException.Normalize()); } diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index f1f667961..e98461cbb 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -102,8 +102,7 @@ import clr clr.AddReference('{path}') "; - var error = Assert.Throws(() => PythonEngine.Exec(code)); - Assert.AreEqual(nameof(FileLoadException), error.PythonTypeName); + Assert.Throws(() => PythonEngine.Exec(code)); } } } diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 66a9a3f7c..1683aeca1 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -160,7 +160,7 @@ public static void TestRunExitFuncs() string msg = e.ToString(); Runtime.Runtime.Shutdown(); - if (e.IsMatches(Exceptions.ImportError)) + if (e.Is(Exceptions.ImportError)) { Assert.Ignore("no atexit module"); } diff --git a/src/runtime/StolenReference.cs b/src/runtime/StolenReference.cs index fb789eec5..1130cff06 100644 --- a/src/runtime/StolenReference.cs +++ b/src/runtime/StolenReference.cs @@ -1,18 +1,46 @@ namespace Python.Runtime { using System; + using System.Diagnostics.Contracts; /// - /// Should only be used for the arguments of Python C API functions, that steal references + /// Should only be used for the arguments of Python C API functions, that steal references, + /// and internal constructors. /// [NonCopyable] readonly ref struct StolenReference { - readonly IntPtr pointer; + internal readonly IntPtr Pointer; internal StolenReference(IntPtr pointer) { - this.pointer = pointer; + Pointer = pointer; } + + [Pure] + public static bool operator ==(in StolenReference reference, NullOnly @null) + => reference.Pointer == IntPtr.Zero; + [Pure] + public static bool operator !=(in StolenReference reference, NullOnly @null) + => reference.Pointer != IntPtr.Zero; + + [Pure] + public override bool Equals(object obj) + { + if (obj is IntPtr ptr) + return ptr == Pointer; + + return false; + } + + [Pure] + public override int GetHashCode() => Pointer.GetHashCode(); + } + + static class StolenReferenceExtensions + { + [Pure] + public static IntPtr DangerousGetAddressOrNull(this in StolenReference reference) + => reference.Pointer; } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 306962f56..d118fc273 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -131,7 +131,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) } else if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } // We modified the Type object, notify it we did. diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index cd9477a62..235b71528 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -110,6 +110,9 @@ internal static IntPtr ToPython(T value) return ToPython(value, typeof(T)); } + internal static NewReference ToPythonReference(T value) + => NewReference.DangerousFromPointer(ToPython(value, typeof(T))); + private static readonly Func IsTransparentProxy = GetIsTransparentProxy(); private static bool Never(object _) => false; diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 440f0eef3..22f603400 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -323,7 +323,7 @@ private object TrueDispatch(object[] args) if (!Converter.ToManaged(op, t, out object newArg, true)) { Exceptions.RaiseTypeError($"The Python function did not return {t.GetElementType()} (the out parameter type)"); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } args[i] = newArg; break; @@ -343,7 +343,7 @@ private object TrueDispatch(object[] args) if (!Converter.ToManaged(item, t, out object newArg, true)) { Exceptions.RaiseTypeError($"The Python function returned a tuple where element {i} was not {t.GetElementType()} (the out parameter type)"); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } args[i] = newArg; } @@ -356,7 +356,7 @@ private object TrueDispatch(object[] args) if (!Converter.ToManaged(item0, rtype, out object result0, true)) { Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)"); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return result0; } @@ -380,7 +380,7 @@ private object TrueDispatch(object[] args) } string returnValueString = isVoid ? "" : "the return value and "; Exceptions.RaiseTypeError($"Expected a tuple ({sb}) of {returnValueString}the values for out and ref parameters, got {tpName}."); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -392,7 +392,7 @@ private object TrueDispatch(object[] args) object result; if (!Converter.ToManaged(op, rtype, out result, true)) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return result; diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 733a2ba80..2647a41c0 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -94,7 +94,7 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public static class Exceptions + internal static class Exceptions { internal static PyModule warnings_module; internal static PyModule exceptions_module; @@ -225,19 +225,6 @@ public static bool ExceptionMatches(IntPtr ob) return Runtime.PyErr_ExceptionMatches(ob) != 0; } - /// - /// ExceptionMatches Method - /// - /// - /// Returns true if the given Python exception matches the given - /// Python object. This is a wrapper for PyErr_GivenExceptionMatches. - /// - public static bool ExceptionMatches(IntPtr exc, IntPtr ob) - { - int i = Runtime.PyErr_GivenExceptionMatches(exc, ob); - return i != 0; - } - /// /// SetError Method /// @@ -271,7 +258,7 @@ public static void SetError(IntPtr type, IntPtr exceptionObject) /// object. The CLR exception instance is wrapped as a Python /// object, allowing it to be handled naturally from Python. /// - public static void SetError(Exception e) + public static bool SetError(Exception e) { // Because delegates allow arbitrary nesting of Python calling // managed calling Python calling... etc. it is possible that we @@ -282,32 +269,33 @@ public static void SetError(Exception e) if (pe != null) { pe.Restore(); - return; + return true; } - IntPtr op = Converter.ToPython(e); - IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__); + using var instance = Converter.ToPythonReference(e); + if (instance.IsNull()) return false; + var exceptionInfo = ExceptionDispatchInfo.Capture(e); - IntPtr pyInfo = Converter.ToPython(exceptionInfo); - Runtime.PyObject_SetAttrString(op, DispatchInfoAttribute, pyInfo); - Runtime.XDecref(pyInfo); + using var pyInfo = Converter.ToPythonReference(exceptionInfo); - Runtime.PyErr_SetObject(etype, op); - Runtime.XDecref(etype); - Runtime.XDecref(op); + if (Runtime.PyObject_SetAttrString(instance, DispatchInfoAttribute, pyInfo) != 0) + return false; + + var type = Runtime.PyObject_TYPE(instance); + Runtime.PyErr_SetObject(type, instance); + return true; } /// /// When called after SetError, sets the cause of the error. /// /// The cause of the current error - public static void SetCause(PythonException cause) + public static void SetCause(Exception cause) { - var currentException = new PythonException(); + var currentException = PythonException.FetchCurrentRaw(); currentException.Normalize(); - cause.Normalize(); - Runtime.XIncref(cause.PyValue); - Runtime.PyException_SetCause(currentException.PyValue, cause.PyValue); + using var causeInstance = Converter.ToPythonReference(cause); + Runtime.PyException_SetCause(currentException.Value.Reference, causeInstance); currentException.Restore(); } @@ -394,16 +382,19 @@ public static void deprecation(string message) /// IntPtr.Zero internal static IntPtr RaiseTypeError(string message) { - PythonException previousException = null; - if (ErrorOccurred()) - { - previousException = new PythonException(); - } + var cause = PythonException.FetchCurrentOrNullRaw(); + cause?.Normalize(); + Exceptions.SetError(Exceptions.TypeError, message); - if (previousException != null) - { - SetCause(previousException); - } + + if (cause is null) return IntPtr.Zero; + + var typeError = PythonException.FetchCurrentRaw(); + typeError.Normalize(); + + Runtime.PyException_SetCause(typeError.Value.Reference, cause.Value.Reference); + typeError.Restore(); + return IntPtr.Zero; } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index fe2e46aac..afc0f8121 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -160,7 +160,7 @@ private void DisposeAll() { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType, errVal, traceback); + Runtime.PyErr_Restore(errType.Steal(), errVal.Steal(), traceback.Steal()); } } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 184b588ad..cef8138ad 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -294,9 +294,7 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) return IntPtr.Zero; } // Save the exception - var originalException = new PythonException(); - // Otherwise, just clear the it. - Exceptions.Clear(); + var originalException = PythonException.FetchCurrentRaw(); string[] names = realname.Split('.'); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 8f74e0052..6b0976b97 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -438,7 +438,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth outs: out outs); if (margs == null) { - mismatchedMethods.Add(new MismatchedMethod(new PythonException(), mi)); + mismatchedMethods.Add(new MismatchedMethod(PythonException.FetchCurrentRaw(), mi)); Exceptions.Clear(); continue; } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 41167e322..54761f6df 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -153,7 +153,7 @@ private void StoreAttribute(string name, ManagedType ob) { if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } ob.IncrRefCount(); cache[name] = ob; @@ -351,7 +351,7 @@ protected override void OnSave(InterDomainContext context) } else if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } pair.Value.DecrRefCount(); } diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs index cf657a033..9fe22aca7 100644 --- a/src/runtime/pybuffer.cs +++ b/src/runtime/pybuffer.cs @@ -18,7 +18,7 @@ unsafe internal PyBuffer(PyObject exporter, PyBUF flags) if (Runtime.PyObject_GetBuffer(exporter.Handle, ref _view, (int)flags) < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } _exporter = exporter; @@ -127,7 +127,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) throw new NotSupportedException("FromContiguous requires at least Python 3.7"); if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// @@ -141,7 +141,7 @@ public void ToContiguous(IntPtr buf, BufferOrderStyle order) throw new ObjectDisposedException(nameof(PyBuffer)); if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// @@ -167,7 +167,7 @@ public void FillInfo(IntPtr exporter, IntPtr buf, long len, bool _readonly, int if (disposedValue) throw new ObjectDisposedException(nameof(PyBuffer)); if (Runtime.PyBuffer_FillInfo(ref _view, exporter, buf, (IntPtr)len, Convert.ToInt32(_readonly), flags) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index 2016ef4f8..da2a600c6 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -57,7 +57,7 @@ public bool MoveNext() { if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } // stop holding the previous object, if there was one diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 7dbb1c690..72e61e759 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -29,9 +29,6 @@ public partial class PyObject : DynamicObject, IEnumerable, IDisposabl public static PyObject None => new PyObject(new BorrowedReference(Runtime.PyNone)); internal BorrowedReference Reference => new BorrowedReference(this.obj); - internal NewReference MakeNewReferenceOrNull() - => NewReference.DangerousFromPointer( - this.obj == IntPtr.Zero ? IntPtr.Zero : Runtime.SelfIncRef(this.obj)); /// /// PyObject Constructor @@ -82,6 +79,17 @@ internal PyObject(BorrowedReference reference) #endif } + internal PyObject(StolenReference reference) + { + if (reference == null) throw new ArgumentNullException(nameof(reference)); + + obj = reference.DangerousGetAddressOrNull(); + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif + } + // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. ~PyObject() @@ -147,7 +155,8 @@ public object AsManagedObject(Type t) object result; if (!Converter.ToManaged(obj, t, out result, true)) { - throw new InvalidCastException("cannot convert object to target type", new PythonException()); + throw new InvalidCastException("cannot convert object to target type", + PythonException.FetchCurrentOrNull(out _)); } return result; } @@ -637,7 +646,7 @@ public virtual long Length() var s = Runtime.PyObject_Size(Reference); if (s < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return s; } @@ -1455,4 +1464,13 @@ public override IEnumerable GetDynamicMemberNames() } } } + + internal static class PyObjectExtensions + { + internal static NewReference NewReferenceOrNull(this PyObject self) + => NewReference.DangerousFromPointer( + (self?.obj ?? IntPtr.Zero) == IntPtr.Zero + ? IntPtr.Zero + : Runtime.SelfIncRef(self.obj)); + } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c5f5a65c4..52cc4e5e6 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -523,7 +523,7 @@ public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = nu using PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.File); if (result.obj != Runtime.PyNone) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } /// @@ -776,7 +776,8 @@ public static void With(PyObject obj, Action Body) PyObject type = PyObject.None; PyObject val = PyObject.None; PyObject traceBack = PyObject.None; - PythonException ex = null; + Exception ex = null; + PythonException pyError = null; try { @@ -785,13 +786,20 @@ public static void With(PyObject obj, Action Body) Body(enterResult); } catch (PythonException e) + { + ex = pyError = e; + } + catch (Exception e) { ex = e; - type = ex.PyType ?? type; - val = ex.PyValue ?? val; - traceBack = ex.PyTB ?? traceBack; + Exceptions.SetError(e); + pyError = PythonException.FetchCurrentRaw(); } + type = pyError?.Type ?? type; + val = pyError?.Value ?? val; + traceBack = pyError?.Traceback ?? traceBack; + var exitResult = obj.InvokeMethod("__exit__", type, val, traceBack); if (ex != null && !exitResult.IsTrue()) throw ex; diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index f11f358d9..fdd22399d 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -12,49 +12,22 @@ namespace Python.Runtime /// public class PythonException : System.Exception { - private PyObject _type; - private PyObject _value; - private PyObject _pyTB; - private string _traceback = ""; - private string _message = ""; - private string _pythonTypeName = ""; - private bool disposed = false; - - [Obsolete("Please, use ThrowLastAsClrException or FromPyErr instead")] - public PythonException() - { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(out var type, out var value, out var traceback); - _type = type.MoveToPyObjectOrNull(); - _value = value.MoveToPyObjectOrNull(); - _pyTB = traceback.MoveToPyObjectOrNull(); - if (_type != null && _value != null) - { - using (PyObject pyTypeName = _type.GetAttr("__name__")) - { - _pythonTypeName = pyTypeName.ToString(); - } - _message = _pythonTypeName + " : " + _value; - } - if (_pyTB != null) - { - _traceback = TracebackToString(_pyTB); - } - PythonEngine.ReleaseLock(gs); + private PythonException(PyType type, PyObject value, PyObject traceback, + Exception innerException) + : base("An exception has occurred in Python code", innerException) + { + Type = type; + Value = value; + Traceback = traceback; } - private PythonException(PyObject type, PyObject value, PyObject traceback, - string message, string pythonTypeName, string tracebackText, - Exception innerException) - : base(message, innerException) + private PythonException(PyType type, PyObject value, PyObject traceback) + : base("An exception has occurred in Python code") { - _type = type; - _value = value; - _pyTB = traceback; - _message = message; - _pythonTypeName = pythonTypeName ?? _pythonTypeName; - _traceback = tracebackText ?? _traceback; + Type = type; + Value = value; + Traceback = traceback; } /// @@ -64,6 +37,47 @@ private PythonException(PyObject type, PyObject value, PyObject traceback, /// internal static Exception ThrowLastAsClrException() { + var exception = FetchCurrentOrNull(out ExceptionDispatchInfo dispatchInfo) + ?? throw new InvalidOperationException("No exception is set"); + dispatchInfo?.Throw(); + // when dispatchInfo is not null, this line will not be reached + throw exception; + } + + internal static PythonException FetchCurrentOrNullRaw() + { + IntPtr gs = PythonEngine.AcquireLock(); + try + { + Runtime.PyErr_Fetch(type: out var type, val: out var value, tb: out var traceback); + if (type.IsNull() && value.IsNull()) + return null; + + try + { + return new PythonException( + type: new PyType(type.Steal()), + value: value.MoveToPyObjectOrNull(), + traceback: traceback.MoveToPyObjectOrNull()); + } + finally + { + type.Dispose(); + } + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + internal static PythonException FetchCurrentRaw() + => FetchCurrentOrNullRaw() + ?? throw new InvalidOperationException("No exception is set"); + + internal static Exception FetchCurrentOrNull(out ExceptionDispatchInfo dispatchInfo) + { + dispatchInfo = default; + // prevent potential interop errors in this method // from crashing process with undebuggable StackOverflowException RuntimeHelpers.EnsureSufficientExecutionStack(); @@ -72,28 +86,27 @@ internal static Exception ThrowLastAsClrException() try { Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + if (value.IsNull() && type.IsNull()) return null; + try { -#if NETSTANDARD if (!value.IsNull()) { - var exceptionInfo = TryGetDispatchInfo(value); - if (exceptionInfo != null) + dispatchInfo = TryGetDispatchInfo(value); + if (dispatchInfo != null) { - exceptionInfo.Throw(); - throw exceptionInfo.SourceException; // unreachable + return dispatchInfo.SourceException; } } -#endif var clrObject = ManagedType.GetManagedObject(value) as CLRObject; if (clrObject?.inst is Exception e) { - throw e; + return e; } var result = FromPyErr(type, value, traceback); - throw result; + return result; } finally { @@ -108,8 +121,7 @@ internal static Exception ThrowLastAsClrException() } } -#if NETSTANDARD - static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference exception) + private static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference exception) { if (exception.IsNull) return null; @@ -137,15 +149,13 @@ static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference exception) pyInfo.Dispose(); } } -#endif /// /// Requires lock to be acquired elsewhere /// - static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference valueHandle, BorrowedReference tracebackHandle) + private static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference valueHandle, BorrowedReference tracebackHandle) { Exception inner = null; - string pythonTypeName = null, msg = "", tracebackText = null; var exceptionDispatchInfo = TryGetDispatchInfo(valueHandle); if (exceptionDispatchInfo != null) @@ -153,13 +163,13 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value return exceptionDispatchInfo.SourceException; } - var clrObject = ManagedType.GetManagedObject(valueHandle) as CLRObject; - if (clrObject?.inst is Exception e) + if (valueHandle != null + && ManagedType.GetManagedObject(valueHandle) is CLRObject { inst: Exception e }) { return e; } - var type = PyObject.FromNullableReference(typeHandle); + var type = PyType.FromNullableReference(typeHandle); var value = PyObject.FromNullableReference(valueHandle); var traceback = PyObject.FromNullableReference(tracebackHandle); @@ -171,43 +181,57 @@ static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference value return decodedException; } - using (PyObject pyTypeName = type.GetAttr("__name__")) - { - pythonTypeName = pyTypeName.ToString(); - } + var raw = new PythonException(type, value, traceback); + raw.Normalize(); - var cause = value.GetAttr("__cause__", null); - if (cause != null && cause.Handle != Runtime.PyNone) + using var cause = Runtime.PyException_GetCause(raw.Value.Reference); + if (!cause.IsNull() && !cause.IsNone()) { - using (var innerTraceback = cause.GetAttr("__traceback__", null)) - { - inner = FromPyErr( - typeHandle: cause.GetPythonTypeReference(), - valueHandle: cause.Reference, - tracebackHandle: innerTraceback is null - ? BorrowedReference.Null - : innerTraceback.Reference); - } + using var innerTraceback = Runtime.PyException_GetTraceback(cause); + inner = FromPyErr( + typeHandle: Runtime.PyObject_TYPE(cause), + valueHandle: cause, + tracebackHandle: innerTraceback); } } - if (traceback != null) + + return new PythonException(type, value, traceback, inner); + } + + private string GetMessage() => GetMessage(Value, Type); + + private static string GetMessage(PyObject value, PyType type) + { + using var _ = new Py.GILState(); + if (value != null && !value.IsNone()) + { + return value.ToString(); + } + + if (type != null) { - tracebackText = TracebackToString(traceback); + return type.Name; } - return new PythonException(type, value, traceback, - msg, pythonTypeName, tracebackText, inner); + throw new ArgumentException("One of the values must not be null"); } - static string TracebackToString(PyObject traceback) + private static string TracebackToString(PyObject traceback) { if (traceback is null) { throw new ArgumentNullException(nameof(traceback)); - throw new ArgumentNullException(nameof(traceback)); } - _finalized = true; - Finalizer.Instance.AddFinalizedObject(this); + + using var tracebackModule = PyModule.Import("traceback"); + using var stackLines = new PyList(tracebackModule.InvokeMethod("format_tb", traceback)); + stackLines.Reverse(); + var result = new StringBuilder(); + foreach (PyObject stackLine in stackLines) + { + result.Append(stackLine); + } + return result.ToString(); } /// Restores python error. @@ -215,46 +239,27 @@ public void Restore() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Restore( - _type.MakeNewReferenceOrNull().Steal(), - _value.MakeNewReferenceOrNull().Steal(), - _pyTB.MakeNewReferenceOrNull().Steal()); + Type.NewReferenceOrNull().Steal(), + Value.NewReferenceOrNull().Steal(), + Traceback.NewReferenceOrNull().Steal()); PythonEngine.ReleaseLock(gs); } /// - /// PyType Property - /// - /// /// Returns the exception type as a Python object. - /// - public PyObject PyType => _type; - - /// - /// PyValue Property /// - /// - /// Returns the exception value as a Python object. - /// - public PyObject PyValue => _value; + public PyType Type { get; private set; } /// - /// PyTB Property + /// Returns the exception value as a Python object. /// - /// - /// Returns the TraceBack as a Python object. - /// - public PyObject PyTB => _pyTB; + /// + public PyObject Value { get; private set; } - /// - /// Message Property - /// /// - /// A string representing the python exception message. + /// Returns the TraceBack as a Python object. /// - public override string Message - { - get { return _message; } - } + public PyObject Traceback { get; } /// /// StackTrace Property @@ -263,20 +268,14 @@ public override string Message /// A string representing the python exception stack trace. /// public override string StackTrace - { - get { return _tb + base.StackTrace; } - } + => (Traceback is null ? "" : TracebackToString(Traceback)) + + base.StackTrace; - /// - /// Python error type name. - /// - public string PythonTypeName - { - get { return _pythonTypeName; } - } + public override string Message => GetMessage(); /// - /// Replaces PyValue with an instance of PyType, if PyValue is not already an instance of PyType. + /// Replaces Value with an instance of Type, if Value is not already an instance of Type. + /// public void Normalize() { IntPtr gs = PythonEngine.AcquireLock(); @@ -284,7 +283,18 @@ public void Normalize() { if (Exceptions.ErrorOccurred()) throw new InvalidOperationException("Cannot normalize when an error is set"); // If an error is set and this PythonException is unnormalized, the error will be cleared and the PythonException will be replaced by a different error. - Runtime.PyErr_NormalizeException(ref _pyType, ref _pyValue, ref _pyTB); + NewReference value = Value.NewReferenceOrNull(); + NewReference type = Type.NewReferenceOrNull(); + NewReference tb = Traceback.NewReferenceOrNull(); + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref tb); + Value = value.MoveToPyObject(); + Type = new PyType(type.Steal()); + if (!tb.IsNull()) + { + int r = Runtime.PyException_SetTraceback(Value.Reference, tb); + ThrowIfIsNotZero(r); + } + tb.Dispose(); } finally { @@ -302,30 +312,19 @@ public string Format() IntPtr gs = PythonEngine.AcquireLock(); try { - if (_pyTB != null && _type != null && _value != null) + var copy = Clone(); + copy.Normalize(); + + if (copy.Traceback != null && copy.Type != null && copy.Value != null) { - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); - using (PyObject pyType = new PyObject(_pyType)) - using (PyObject pyValue = new PyObject(_pyValue)) - using (PyObject pyTB = new PyObject(_pyTB)) - using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + using var traceback = PyModule.Import("traceback"); + var buffer = new StringBuilder(); + var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); + foreach (PyObject val in values) { - var buffer = new StringBuilder(); - var values = tb_mod.InvokeMethod("format_exception", _type, _value, _pyTB); - foreach (PyObject val in values) - { - buffer.Append(val.ToString()); - } - res = buffer.ToString(); - var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); - foreach (PyObject val in values) - { - buffer.Append(val.ToString()); - } - res = buffer.ToString(); + buffer.Append(val); } + res = buffer.ToString(); } else { @@ -339,45 +338,36 @@ public string Format() return res; } - public bool IsMatches(IntPtr exc) + public PythonException Clone() + => new PythonException(Type, Value, Traceback, InnerException); + + internal bool Is(IntPtr type) { - return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; + return Runtime.PyErr_GivenExceptionMatches( + (Value ?? Type).Reference, + new BorrowedReference(type)) != 0; } /// - /// Dispose Method + /// Returns true if the current Python exception + /// matches the given exception type. /// - /// - /// The Dispose method provides a way to explicitly release the - /// Python objects represented by a PythonException. - /// If object not properly disposed can cause AppDomain unload issue. - /// See GH#397 and GH#400. - /// - public void Dispose() + internal static bool CurrentMatches(IntPtr ob) { - if (!disposed) - { - _type?.Dispose(); - _value?.Dispose(); - _pyTB?.Dispose(); - GC.SuppressFinalize(this); - disposed = true; - } + return Runtime.PyErr_ExceptionMatches(ob) != 0; } - /// - /// Matches Method - /// - /// - /// Returns true if the Python exception type represented by the - /// PythonException instance matches the given exception type. - /// - internal static bool Matches(IntPtr ob) + internal static BorrowedReference ThrowIfIsNull(BorrowedReference ob) { - return Runtime.PyErr_ExceptionMatches(ob) != 0; + if (ob == null) + { + throw ThrowLastAsClrException(); + } + + return ob; } - public static void ThrowIfIsNull(IntPtr ob) + public static IntPtr ThrowIfIsNull(IntPtr ob) { if (ob == IntPtr.Zero) { diff --git a/src/runtime/pytuple.cs b/src/runtime/pytuple.cs index 530ced3d2..5a18b6bed 100644 --- a/src/runtime/pytuple.cs +++ b/src/runtime/pytuple.cs @@ -77,7 +77,7 @@ private static IntPtr FromArray(PyObject[] items) if (res != 0) { Runtime.Py_DecRef(val); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } return val; diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs index 8bc08b76d..85ee54171 100644 --- a/src/runtime/pytype.cs +++ b/src/runtime/pytype.cs @@ -13,6 +13,27 @@ public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases) /// Wraps an existing type object. public PyType(PyObject o) : base(FromObject(o)) { } + internal PyType(StolenReference reference) : base(EnsureIsType(in reference)) + { + } + + internal new static PyType? FromNullableReference(BorrowedReference reference) + => reference == null + ? null + : new PyType(new NewReference(reference).Steal()); + + public string Name + { + get + { + var namePtr = new StrPtr + { + RawPointer = Marshal.ReadIntPtr(Handle, TypeOffset.tp_name), + }; + return namePtr.ToString(System.Text.Encoding.UTF8)!; + } + } + /// Checks if specified object is a Python type. public static bool IsType(PyObject value) { @@ -21,6 +42,19 @@ public static bool IsType(PyObject value) return Runtime.PyType_Check(value.obj); } + private static IntPtr EnsureIsType(in StolenReference reference) + { + IntPtr address = reference.DangerousGetAddressOrNull(); + if (address == IntPtr.Zero) + throw new ArgumentNullException(nameof(reference)); + return EnsureIsType(address); + } + + private static IntPtr EnsureIsType(IntPtr ob) + => Runtime.PyType_Check(ob) + ? ob + : throw new ArgumentException("object is not a type"); + private static BorrowedReference FromObject(PyObject o) { if (o is null) throw new ArgumentNullException(nameof(o)); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 3ce000b5b..194e928c0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -426,11 +426,10 @@ private static void RunExitFuncs() } catch (PythonException e) { - if (!e.IsMatches(Exceptions.ImportError)) + if (!e.Is(Exceptions.ImportError)) { throw; } - e.Dispose(); // The runtime may not provided `atexit` module. return; } @@ -443,7 +442,6 @@ private static void RunExitFuncs() catch (PythonException e) { Console.Error.WriteLine(e); - e.Dispose(); } } } @@ -492,9 +490,9 @@ private static void PyDictTryDelItem(BorrowedReference dict, string key) { return; } - if (!PythonException.Matches(Exceptions.KeyError)) + if (!PythonException.CurrentMatches(Exceptions.KeyError)) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } PyErr_Clear(); } @@ -1045,6 +1043,11 @@ internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr v using var namePtr = new StrPtr(name, Encoding.UTF8); return Delegates.PyObject_SetAttrString(pointer, namePtr, value); } + internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(@object.DangerousGetAddress(), namePtr, value.DangerousGetAddress()); + } internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); @@ -2085,19 +2088,19 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static int PyErr_ExceptionMatches(IntPtr exception) => Delegates.PyErr_ExceptionMatches(exception); - internal static int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val) => Delegates.PyErr_GivenExceptionMatches(ob, val); + internal static int PyErr_GivenExceptionMatches(BorrowedReference ob, BorrowedReference val) => Delegates.PyErr_GivenExceptionMatches(ob, val); - internal static void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb) => Delegates.PyErr_NormalizeException(ref ob, ref val, ref tb); + internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); internal static IntPtr PyErr_Occurred() => Delegates.PyErr_Occurred(); - internal static void PyErr_Fetch(out NewReference ob, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out ob, out val, out tb); + internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); - internal static void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb) => Delegates.PyErr_Restore(ob, val, tb); + internal static void PyErr_Restore(StolenReference ob, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(ob, val, tb); internal static void PyErr_Clear() => Delegates.PyErr_Clear(); @@ -2105,11 +2108,19 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static void PyErr_Print() => Delegates.PyErr_Print(); + + internal static NewReference PyException_GetCause(BorrowedReference ex) + => Delegates.PyException_GetCause(ex); + internal static NewReference PyException_GetTraceback(BorrowedReference ex) + => Delegates.PyException_GetTraceback(ex); + /// /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. /// - - internal static void PyException_SetCause(IntPtr ex, IntPtr cause) => Delegates.PyException_SetCause(ex, cause); + internal static void PyException_SetCause(BorrowedReference ex, BorrowedReference cause) + => Delegates.PyException_SetCause(ex, cause); + internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) + => Delegates.PyException_SetTraceback(ex, tb); //==================================================================== // Cell API @@ -2487,11 +2498,11 @@ static Delegates() PyErr_SetFromErrno = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetFromErrno), GetUnmanagedDll(_PythonDll)); PyErr_SetNone = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetNone), GetUnmanagedDll(_PythonDll)); PyErr_ExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_ExceptionMatches), GetUnmanagedDll(_PythonDll)); - PyErr_GivenExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_GivenExceptionMatches), GetUnmanagedDll(_PythonDll)); - PyErr_NormalizeException = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_NormalizeException), GetUnmanagedDll(_PythonDll)); + PyErr_GivenExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_GivenExceptionMatches), GetUnmanagedDll(_PythonDll)); + PyErr_NormalizeException = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_NormalizeException), GetUnmanagedDll(_PythonDll)); PyErr_Occurred = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Occurred), GetUnmanagedDll(_PythonDll)); PyErr_Fetch = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Fetch), GetUnmanagedDll(_PythonDll)); - PyErr_Restore = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Restore), GetUnmanagedDll(_PythonDll)); + PyErr_Restore = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Restore), GetUnmanagedDll(_PythonDll)); PyErr_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Clear), GetUnmanagedDll(_PythonDll)); PyErr_Print = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Print), GetUnmanagedDll(_PythonDll)); PyCell_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Get), GetUnmanagedDll(_PythonDll)); @@ -2508,7 +2519,10 @@ static Delegates() PyLong_AsSignedSize_t = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsSsize_t", GetUnmanagedDll(_PythonDll)); PyExplicitlyConvertToInt64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLongLong", GetUnmanagedDll(_PythonDll)); PyDict_GetItemWithError = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemWithError), GetUnmanagedDll(_PythonDll)); - PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); + PyException_GetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetCause), GetUnmanagedDll(_PythonDll)); + PyException_GetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetTraceback), GetUnmanagedDll(_PythonDll)); + PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); + PyException_SetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetTraceback), GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); @@ -2764,11 +2778,11 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyErr_SetFromErrno { get; } internal static delegate* unmanaged[Cdecl] PyErr_SetNone { get; } internal static delegate* unmanaged[Cdecl] PyErr_ExceptionMatches { get; } - internal static delegate* unmanaged[Cdecl] PyErr_GivenExceptionMatches { get; } - internal static delegate* unmanaged[Cdecl] PyErr_NormalizeException { get; } + internal static delegate* unmanaged[Cdecl] PyErr_GivenExceptionMatches { get; } + internal static delegate* unmanaged[Cdecl] PyErr_NormalizeException { get; } internal static delegate* unmanaged[Cdecl] PyErr_Occurred { get; } internal static delegate* unmanaged[Cdecl] PyErr_Fetch { get; } - internal static delegate* unmanaged[Cdecl] PyErr_Restore { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Restore { get; } internal static delegate* unmanaged[Cdecl] PyErr_Clear { get; } internal static delegate* unmanaged[Cdecl] PyErr_Print { get; } internal static delegate* unmanaged[Cdecl] PyCell_Get { get; } @@ -2785,7 +2799,10 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyLong_AsSignedSize_t { get; } internal static delegate* unmanaged[Cdecl] PyExplicitlyConvertToInt64 { get; } internal static delegate* unmanaged[Cdecl] PyDict_GetItemWithError { get; } - internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_GetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_GetTraceback { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetTraceback { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 01aceb656..fde8fb719 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -177,7 +177,7 @@ internal static IntPtr CreateType(Type impl) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); @@ -300,7 +300,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); @@ -472,7 +472,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); @@ -577,7 +577,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); @@ -926,14 +926,14 @@ public static IntPtr CreateObjectType() if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) { globals.Dispose(); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } const string code = "class A(object): pass"; using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); if (resRef.IsNull()) { globals.Dispose(); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } resRef.Dispose(); BorrowedReference A = Runtime.PyDict_GetItemString(globals, "A"); From e58411d90547cb3da22697c09edb6f190e0469ad Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 9 Apr 2021 12:08:05 -0700 Subject: [PATCH 086/151] rum embedding tests before Python tests --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2dd75c529..0afb6016a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,6 +56,9 @@ jobs: run: | python -m pythonnet.find_libpython --export | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Embedding tests + run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ + - name: Python Tests (Mono) if: ${{ matrix.os != 'windows' }} run: pytest --runtime mono @@ -67,9 +70,6 @@ jobs: if: ${{ matrix.os == 'windows' }} run: pytest --runtime netfx - - name: Embedding tests - run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ - - name: Python tests run from .NET run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ From 00653dcf9a72b814a45a47588750f4ef32fb5497 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 9 Apr 2021 12:40:58 -0700 Subject: [PATCH 087/151] PythonException.StackTrace is GIL-safe --- src/runtime/pythonexception.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index fdd22399d..0912b828e 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -268,8 +268,15 @@ public void Restore() /// A string representing the python exception stack trace. /// public override string StackTrace - => (Traceback is null ? "" : TracebackToString(Traceback)) - + base.StackTrace; + { + get + { + if (Traceback is null) return base.StackTrace; + + using var _ = new Py.GILState(); + return TracebackToString(Traceback) + base.StackTrace; + } + } public override string Message => GetMessage(); From 343320139f410534d5f8f5d80dec99e24b9088e3 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 17:34:30 -0700 Subject: [PATCH 088/151] separate .Steal() and .StealNullable() --- src/runtime/NewReference.cs | 16 +++++++++++++++- src/runtime/finalizer.cs | 2 +- src/runtime/pyobject.cs | 2 +- src/runtime/pythonexception.cs | 6 +++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 3bc5d9724..c037f988f 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -2,6 +2,7 @@ namespace Python.Runtime { using System; using System.Diagnostics.Contracts; + using System.Runtime.CompilerServices; /// /// Represents a reference to a Python object, that is tracked by Python's reference counting. @@ -65,12 +66,25 @@ public IntPtr DangerousMoveToPointerOrNull() /// Call this method to move ownership of this reference to a Python C API function, /// that steals reference passed to it. /// - public StolenReference Steal() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StolenReference StealNullable() { IntPtr rawPointer = this.pointer; this.pointer = IntPtr.Zero; return new StolenReference(rawPointer); } + + /// + /// Call this method to move ownership of this reference to a Python C API function, + /// that steals reference passed to it. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StolenReference Steal() + { + if (this.IsNull()) throw new NullReferenceException(); + + return this.StealNullable(); + } /// /// Removes this reference to a Python object, and sets it to null. /// diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index afc0f8121..91a4b859e 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -160,7 +160,7 @@ private void DisposeAll() { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType.Steal(), errVal.Steal(), traceback.Steal()); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 72e61e759..65dea3665 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -218,7 +218,7 @@ protected virtual void Dispose(bool disposing) { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType.Steal(), errVal.Steal(), traceback.Steal()); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } else diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 0912b828e..d80fbb195 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -239,9 +239,9 @@ public void Restore() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Restore( - Type.NewReferenceOrNull().Steal(), - Value.NewReferenceOrNull().Steal(), - Traceback.NewReferenceOrNull().Steal()); + Type.NewReferenceOrNull().StealNullable(), + Value.NewReferenceOrNull().StealNullable(), + Traceback.NewReferenceOrNull().StealNullable()); PythonEngine.ReleaseLock(gs); } From 95cc52fdb221909f8b93e0382a979417ef82d4f8 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 17:34:52 -0700 Subject: [PATCH 089/151] can't test exception type when runtime is down --- src/embed_tests/pyinitialize.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 1683aeca1..1622f46d3 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -158,9 +158,10 @@ public static void TestRunExitFuncs() catch (PythonException e) { string msg = e.ToString(); + bool isImportError = e.Is(Exceptions.ImportError); Runtime.Runtime.Shutdown(); - if (e.Is(Exceptions.ImportError)) + if (isImportError) { Assert.Ignore("no atexit module"); } From 63ad42ce7e91130af178a687a6c4a87d08344035 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 17:35:50 -0700 Subject: [PATCH 090/151] PythonException: dispose intermediate values used in stack trace generation --- src/runtime/pythonexception.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index d80fbb195..35ff84dc2 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -230,6 +230,7 @@ private static string TracebackToString(PyObject traceback) foreach (PyObject stackLine in stackLines) { result.Append(stackLine); + stackLine.Dispose(); } return result.ToString(); } @@ -326,10 +327,11 @@ public string Format() { using var traceback = PyModule.Import("traceback"); var buffer = new StringBuilder(); - var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); + using var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); foreach (PyObject val in values) { buffer.Append(val); + val.Dispose(); } res = buffer.ToString(); } From faec7fc1635cc14416eae49101b2bfb62f3536f4 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 17:36:13 -0700 Subject: [PATCH 091/151] Point users to Message property of PythonException --- src/runtime/pythonexception.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 35ff84dc2..47db8ad2c 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -15,7 +15,7 @@ public class PythonException : System.Exception private PythonException(PyType type, PyObject value, PyObject traceback, Exception innerException) - : base("An exception has occurred in Python code", innerException) + : base("An exception has occurred in Python code. See Message property for details.", innerException) { Type = type; Value = value; @@ -23,7 +23,7 @@ private PythonException(PyType type, PyObject value, PyObject traceback, } private PythonException(PyType type, PyObject value, PyObject traceback) - : base("An exception has occurred in Python code") + : base("An exception has occurred in Python code. See Message property for details.") { Type = type; Value = value; From dfc70f682287b0c68d7160d4b37df2361c02bb35 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 19:33:49 -0700 Subject: [PATCH 092/151] minor change in PythonEngine.With --- src/runtime/pythonengine.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 52cc4e5e6..419d4554a 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -773,9 +773,6 @@ public static void With(PyObject obj, Action Body) // Behavior described here: // https://docs.python.org/2/reference/datamodel.html#with-statement-context-managers - PyObject type = PyObject.None; - PyObject val = PyObject.None; - PyObject traceBack = PyObject.None; Exception ex = null; PythonException pyError = null; @@ -796,9 +793,9 @@ public static void With(PyObject obj, Action Body) pyError = PythonException.FetchCurrentRaw(); } - type = pyError?.Type ?? type; - val = pyError?.Value ?? val; - traceBack = pyError?.Traceback ?? traceBack; + PyObject type = pyError?.Type ?? PyObject.None; + PyObject val = pyError?.Value ?? PyObject.None; + PyObject traceBack = pyError?.Traceback ?? PyObject.None; var exitResult = obj.InvokeMethod("__exit__", type, val, traceBack); From d976acf44de2e6d1c67929522b25b7528c747501 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 19:37:37 -0700 Subject: [PATCH 093/151] simplified code of PythonException and added a lot more checks --- src/runtime/exceptions.cs | 2 +- src/runtime/pythonexception.cs | 287 +++++++++++++++++---------------- src/runtime/pytype.cs | 3 + src/runtime/runtime.cs | 19 +-- 4 files changed, 159 insertions(+), 152 deletions(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 2647a41c0..669ef9e6f 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -308,7 +308,7 @@ public static void SetCause(Exception cause) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != null; } /// diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 47db8ad2c..3f4bd8154 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,6 @@ +#nullable enable using System; +using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; @@ -13,22 +15,17 @@ namespace Python.Runtime public class PythonException : System.Exception { - private PythonException(PyType type, PyObject value, PyObject traceback, - Exception innerException) + public PythonException(PyType type, PyObject? value, PyObject? traceback, + Exception? innerException) : base("An exception has occurred in Python code. See Message property for details.", innerException) { - Type = type; + Type = type ?? throw new ArgumentNullException(nameof(type)); Value = value; Traceback = traceback; } - private PythonException(PyType type, PyObject value, PyObject traceback) - : base("An exception has occurred in Python code. See Message property for details.") - { - Type = type; - Value = value; - Traceback = traceback; - } + public PythonException(PyType type, PyObject? value, PyObject? traceback) + : this(type, value, traceback, innerException: null) { } /// /// Rethrows the last Python exception as corresponding CLR exception. @@ -37,91 +34,67 @@ private PythonException(PyType type, PyObject value, PyObject traceback) /// internal static Exception ThrowLastAsClrException() { - var exception = FetchCurrentOrNull(out ExceptionDispatchInfo dispatchInfo) + var exception = FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) ?? throw new InvalidOperationException("No exception is set"); dispatchInfo?.Throw(); // when dispatchInfo is not null, this line will not be reached throw exception; } - internal static PythonException FetchCurrentOrNullRaw() + internal static PythonException? FetchCurrentOrNullRaw() { - IntPtr gs = PythonEngine.AcquireLock(); - try - { - Runtime.PyErr_Fetch(type: out var type, val: out var value, tb: out var traceback); - if (type.IsNull() && value.IsNull()) - return null; + using var _ = new Py.GILState(); - try - { - return new PythonException( - type: new PyType(type.Steal()), - value: value.MoveToPyObjectOrNull(), - traceback: traceback.MoveToPyObjectOrNull()); - } - finally - { - type.Dispose(); - } - } - finally + Runtime.PyErr_Fetch(type: out var type, val: out var value, tb: out var traceback); + + if (type.IsNull()) { - PythonEngine.ReleaseLock(gs); + Debug.Assert(value.IsNull()); + Debug.Assert(traceback.IsNull()); + return null; } + + return new PythonException( + type: new PyType(type.Steal()), + value: value.MoveToPyObjectOrNull(), + traceback: traceback.MoveToPyObjectOrNull()); } internal static PythonException FetchCurrentRaw() => FetchCurrentOrNullRaw() ?? throw new InvalidOperationException("No exception is set"); - internal static Exception FetchCurrentOrNull(out ExceptionDispatchInfo dispatchInfo) + internal static Exception? FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) { - dispatchInfo = default; + dispatchInfo = null; // prevent potential interop errors in this method // from crashing process with undebuggable StackOverflowException RuntimeHelpers.EnsureSufficientExecutionStack(); - IntPtr gs = PythonEngine.AcquireLock(); - try + using var _ = new Py.GILState(); + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + if (type.IsNull()) { - Runtime.PyErr_Fetch(out var type, out var value, out var traceback); - if (value.IsNull() && type.IsNull()) return null; - - try - { - if (!value.IsNull()) - { - dispatchInfo = TryGetDispatchInfo(value); - if (dispatchInfo != null) - { - return dispatchInfo.SourceException; - } - } + Debug.Assert(value.IsNull()); + Debug.Assert(traceback.IsNull()); + return null; + } - var clrObject = ManagedType.GetManagedObject(value) as CLRObject; - if (clrObject?.inst is Exception e) - { - return e; - } + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref traceback); - var result = FromPyErr(type, value, traceback); - return result; - } - finally - { - type.Dispose(); - value.Dispose(); - traceback.Dispose(); - } + try + { + return FromPyErr(typeRef: type, valRef: value, tbRef: traceback, out dispatchInfo); } finally { - PythonEngine.ReleaseLock(gs); + type.Dispose(); + value.Dispose(); + traceback.Dispose(); } } - private static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference exception) + private static ExceptionDispatchInfo? TryGetDispatchInfo(BorrowedReference exception) { if (exception.IsNull) return null; @@ -153,54 +126,53 @@ private static ExceptionDispatchInfo TryGetDispatchInfo(BorrowedReference except /// /// Requires lock to be acquired elsewhere /// - private static Exception FromPyErr(BorrowedReference typeHandle, BorrowedReference valueHandle, BorrowedReference tracebackHandle) + private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef, + out ExceptionDispatchInfo? exceptionDispatchInfo) { - Exception inner = null; + if (valRef == null) throw new ArgumentNullException(nameof(valRef)); + + var type = PyType.FromReference(typeRef); + var value = new PyObject(valRef); + var traceback = PyObject.FromNullableReference(tbRef); - var exceptionDispatchInfo = TryGetDispatchInfo(valueHandle); + exceptionDispatchInfo = TryGetDispatchInfo(valRef); if (exceptionDispatchInfo != null) { return exceptionDispatchInfo.SourceException; } - if (valueHandle != null - && ManagedType.GetManagedObject(valueHandle) is CLRObject { inst: Exception e }) + if (ManagedType.GetManagedObject(valRef) is CLRObject { inst: Exception e }) { return e; } - var type = PyType.FromNullableReference(typeHandle); - var value = PyObject.FromNullableReference(valueHandle); - var traceback = PyObject.FromNullableReference(tracebackHandle); - - if (type != null && value != null) + if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object decoded) + && decoded is Exception decodedException) { - if (PyObjectConversions.TryDecode(valueHandle, typeHandle, typeof(Exception), out object decoded) - && decoded is Exception decodedException) - { - return decodedException; - } - - var raw = new PythonException(type, value, traceback); - raw.Normalize(); - - using var cause = Runtime.PyException_GetCause(raw.Value.Reference); - if (!cause.IsNull() && !cause.IsNone()) - { - using var innerTraceback = Runtime.PyException_GetTraceback(cause); - inner = FromPyErr( - typeHandle: Runtime.PyObject_TYPE(cause), - valueHandle: cause, - tracebackHandle: innerTraceback); - } + return decodedException; } + using var cause = Runtime.PyException_GetCause(valRef); + Exception? inner = FromCause(cause); return new PythonException(type, value, traceback, inner); } - private string GetMessage() => GetMessage(Value, Type); + private static Exception? FromCause(BorrowedReference cause) + { + if (cause == null || cause.IsNone()) return null; - private static string GetMessage(PyObject value, PyType type) + Debug.Assert(Runtime.PyObject_TypeCheck(cause, new BorrowedReference(Exceptions.BaseException))); + + using var innerTraceback = Runtime.PyException_GetTraceback(cause); + return FromPyErr( + typeRef: Runtime.PyObject_TYPE(cause), + valRef: cause, + tbRef: innerTraceback, + out _); + + } + + private static string GetMessage(PyObject? value, PyType type) { using var _ = new Py.GILState(); if (value != null && !value.IsNone()) @@ -208,12 +180,7 @@ private static string GetMessage(PyObject value, PyType type) return value.ToString(); } - if (type != null) - { - return type.Name; - } - - throw new ArgumentException("One of the values must not be null"); + return type.Name; } private static string TracebackToString(PyObject traceback) @@ -238,12 +205,18 @@ private static string TracebackToString(PyObject traceback) /// Restores python error. public void Restore() { - IntPtr gs = PythonEngine.AcquireLock(); + CheckRuntimeIsRunning(); + + using var _ = new Py.GILState(); + + NewReference type = Type.NewReferenceOrNull(); + NewReference value = Value.NewReferenceOrNull(); + NewReference traceback = Traceback.NewReferenceOrNull(); + Runtime.PyErr_Restore( - Type.NewReferenceOrNull().StealNullable(), - Value.NewReferenceOrNull().StealNullable(), - Traceback.NewReferenceOrNull().StealNullable()); - PythonEngine.ReleaseLock(gs); + type: type.Steal(), + val: value.StealNullable(), + tb: traceback.StealNullable()); } /// @@ -255,12 +228,12 @@ public void Restore() /// Returns the exception value as a Python object. /// /// - public PyObject Value { get; private set; } + public PyObject? Value { get; private set; } /// /// Returns the TraceBack as a Python object. /// - public PyObject Traceback { get; } + public PyObject? Traceback { get; } /// /// StackTrace Property @@ -274,35 +247,74 @@ public override string StackTrace { if (Traceback is null) return base.StackTrace; + if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) + return "Python stack unavailable as runtime was shut down\n" + base.StackTrace; + using var _ = new Py.GILState(); return TracebackToString(Traceback) + base.StackTrace; } } - public override string Message => GetMessage(); + public override string Message + { + get + { + if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) + return "Python error message is unavailable as runtime was shut down"; + + return GetMessage(this.Value, this.Type); + } + } + + public bool IsNormalized + { + get + { + if (Value is null) return false; + + CheckRuntimeIsRunning(); + + using var _ = new Py.GILState(); + return Runtime.PyObject_TypeCheck(Value.Reference, Type.Reference); + } + } /// /// Replaces Value with an instance of Type, if Value is not already an instance of Type. /// public void Normalize() { + CheckRuntimeIsRunning(); + IntPtr gs = PythonEngine.AcquireLock(); try { if (Exceptions.ErrorOccurred()) throw new InvalidOperationException("Cannot normalize when an error is set"); + // If an error is set and this PythonException is unnormalized, the error will be cleared and the PythonException will be replaced by a different error. NewReference value = Value.NewReferenceOrNull(); NewReference type = Type.NewReferenceOrNull(); NewReference tb = Traceback.NewReferenceOrNull(); + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref tb); + Value = value.MoveToPyObject(); Type = new PyType(type.Steal()); - if (!tb.IsNull()) + try { - int r = Runtime.PyException_SetTraceback(Value.Reference, tb); - ThrowIfIsNotZero(r); + Debug.Assert(Traceback is null == tb.IsNull()); + if (!tb.IsNull()) + { + Debug.Assert(Traceback!.Reference == tb); + + int r = Runtime.PyException_SetTraceback(Value.Reference, tb); + ThrowIfIsNotZero(r); + } + } + finally + { + tb.Dispose(); } - tb.Dispose(); } finally { @@ -316,35 +328,26 @@ public void Normalize() /// public string Format() { - string res; - IntPtr gs = PythonEngine.AcquireLock(); - try - { - var copy = Clone(); - copy.Normalize(); + CheckRuntimeIsRunning(); - if (copy.Traceback != null && copy.Type != null && copy.Value != null) - { - using var traceback = PyModule.Import("traceback"); - var buffer = new StringBuilder(); - using var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); - foreach (PyObject val in values) - { - buffer.Append(val); - val.Dispose(); - } - res = buffer.ToString(); - } - else - { - res = StackTrace; - } - } - finally + using var _ = new Py.GILState(); + + var copy = Clone(); + copy.Normalize(); + + if (copy.Traceback is null || copy.Value is null) + return StackTrace; + + using var traceback = PyModule.Import("traceback"); + var buffer = new StringBuilder(); + using var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); + foreach (PyObject val in values) { - PythonEngine.ReleaseLock(gs); + buffer.Append(val); + val.Dispose(); } - return res; + return buffer.ToString(); + } public PythonException Clone() @@ -357,6 +360,12 @@ internal bool Is(IntPtr type) new BorrowedReference(type)) != 0; } + private static void CheckRuntimeIsRunning() + { + if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be running"); + } + /// /// Returns true if the current Python exception /// matches the given exception type. diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs index 85ee54171..3ce842a4d 100644 --- a/src/runtime/pytype.cs +++ b/src/runtime/pytype.cs @@ -22,6 +22,9 @@ internal PyType(StolenReference reference) : base(EnsureIsType(in reference)) ? null : new PyType(new NewReference(reference).Steal()); + internal static PyType FromReference(BorrowedReference reference) + => FromNullableReference(reference) ?? throw new ArgumentNullException(nameof(reference)); + public string Name { get diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 194e928c0..e6cf8f13c 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,8 +1,7 @@ -using System.Reflection.Emit; using System; +using System.Diagnostics; using System.Diagnostics.Contracts; using System.Runtime.InteropServices; -using System.Security; using System.Text; using System.Threading; using System.Collections.Generic; @@ -424,12 +423,8 @@ private static void RunExitFuncs() { atexit = Py.Import("atexit"); } - catch (PythonException e) + catch (PythonException e) when (e.Is(Exceptions.ImportError)) { - if (!e.Is(Exceptions.ImportError)) - { - throw; - } // The runtime may not provided `atexit` module. return; } @@ -586,7 +581,7 @@ public static PyObject None /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != null) { throw PythonException.ThrowLastAsClrException(); } @@ -2094,13 +2089,13 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); - internal static IntPtr PyErr_Occurred() => Delegates.PyErr_Occurred(); + internal static BorrowedReference PyErr_Occurred() => Delegates.PyErr_Occurred(); internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); - internal static void PyErr_Restore(StolenReference ob, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(ob, val, tb); + internal static void PyErr_Restore(StolenReference type, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(type, val, tb); internal static void PyErr_Clear() => Delegates.PyErr_Clear(); @@ -2500,7 +2495,7 @@ static Delegates() PyErr_ExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_ExceptionMatches), GetUnmanagedDll(_PythonDll)); PyErr_GivenExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_GivenExceptionMatches), GetUnmanagedDll(_PythonDll)); PyErr_NormalizeException = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_NormalizeException), GetUnmanagedDll(_PythonDll)); - PyErr_Occurred = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Occurred), GetUnmanagedDll(_PythonDll)); + PyErr_Occurred = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Occurred), GetUnmanagedDll(_PythonDll)); PyErr_Fetch = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Fetch), GetUnmanagedDll(_PythonDll)); PyErr_Restore = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Restore), GetUnmanagedDll(_PythonDll)); PyErr_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Clear), GetUnmanagedDll(_PythonDll)); @@ -2780,7 +2775,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyErr_ExceptionMatches { get; } internal static delegate* unmanaged[Cdecl] PyErr_GivenExceptionMatches { get; } internal static delegate* unmanaged[Cdecl] PyErr_NormalizeException { get; } - internal static delegate* unmanaged[Cdecl] PyErr_Occurred { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Occurred { get; } internal static delegate* unmanaged[Cdecl] PyErr_Fetch { get; } internal static delegate* unmanaged[Cdecl] PyErr_Restore { get; } internal static delegate* unmanaged[Cdecl] PyErr_Clear { get; } From 146ebf3f3d069046b13a812f649e8303186cbdf5 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 10 Apr 2021 20:06:24 -0700 Subject: [PATCH 094/151] fixed type of reference in PyException_SetCause --- src/runtime/exceptions.cs | 6 ++++-- src/runtime/runtime.cs | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 669ef9e6f..0540d81ca 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -295,7 +295,7 @@ public static void SetCause(Exception cause) var currentException = PythonException.FetchCurrentRaw(); currentException.Normalize(); using var causeInstance = Converter.ToPythonReference(cause); - Runtime.PyException_SetCause(currentException.Value.Reference, causeInstance); + Runtime.PyException_SetCause(currentException.Value!.Reference, causeInstance.Steal()); currentException.Restore(); } @@ -392,7 +392,9 @@ internal static IntPtr RaiseTypeError(string message) var typeError = PythonException.FetchCurrentRaw(); typeError.Normalize(); - Runtime.PyException_SetCause(typeError.Value.Reference, cause.Value.Reference); + Runtime.PyException_SetCause( + typeError.Value!.Reference, + new NewReference(cause.Value!.Reference).Steal()); typeError.Restore(); return IntPtr.Zero; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e6cf8f13c..9f6b92cac 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2112,7 +2112,7 @@ internal static NewReference PyException_GetTraceback(BorrowedReference ex) /// /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. /// - internal static void PyException_SetCause(BorrowedReference ex, BorrowedReference cause) + internal static void PyException_SetCause(BorrowedReference ex, StolenReference cause) => Delegates.PyException_SetCause(ex, cause); internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) => Delegates.PyException_SetTraceback(ex, tb); @@ -2516,7 +2516,7 @@ static Delegates() PyDict_GetItemWithError = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemWithError), GetUnmanagedDll(_PythonDll)); PyException_GetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetCause), GetUnmanagedDll(_PythonDll)); PyException_GetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetTraceback), GetUnmanagedDll(_PythonDll)); - PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); + PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); PyException_SetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetTraceback), GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); @@ -2796,7 +2796,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyDict_GetItemWithError { get; } internal static delegate* unmanaged[Cdecl] PyException_GetCause { get; } internal static delegate* unmanaged[Cdecl] PyException_GetTraceback { get; } - internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } internal static delegate* unmanaged[Cdecl] PyException_SetTraceback { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } From 272687bfdb97654618701f9eaad770d95993580d Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 01:01:20 -0700 Subject: [PATCH 095/151] minor fixes to Converter.ToArray: - no longer leaking iterator object on failure - when iteration stops due to error, propagates the error --- src/runtime/converter.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 235b71528..645f31daa 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -888,6 +888,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s if (!Converter.ToManaged(item, elementType, out obj, setError)) { Runtime.XDecref(item); + Runtime.XDecref(IterObject); return false; } @@ -896,6 +897,12 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s } Runtime.XDecref(IterObject); + if (Exceptions.ErrorOccurred()) + { + if (!setError) Exceptions.Clear(); + return false; + } + Array items = Array.CreateInstance(elementType, list.Count); list.CopyTo(items, 0); From 2cd3f6189d09e114ddab7aa1204a1d2a86865373 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 01:02:27 -0700 Subject: [PATCH 096/151] added a few debug checks to Exceptions.SetError --- src/runtime/exceptions.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 0540d81ca..6a3f2ec6c 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -1,8 +1,8 @@ using System; +using System.Diagnostics; using System.Reflection; using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; -using System.Text; namespace Python.Runtime { @@ -260,6 +260,8 @@ public static void SetError(IntPtr type, IntPtr exceptionObject) /// public static bool SetError(Exception e) { + Debug.Assert(e is not null); + // Because delegates allow arbitrary nesting of Python calling // managed calling Python calling... etc. it is possible that we // might get a managed exception raised that is a wrapper for a @@ -281,6 +283,8 @@ public static bool SetError(Exception e) if (Runtime.PyObject_SetAttrString(instance, DispatchInfoAttribute, pyInfo) != 0) return false; + Debug.Assert(Runtime.PyObject_TypeCheck(instance, new BorrowedReference(BaseException))); + var type = Runtime.PyObject_TYPE(instance); Runtime.PyErr_SetObject(type, instance); return true; From e79f041f3a72e1855e8b0239ccadf954f58d9fec Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 01:03:38 -0700 Subject: [PATCH 097/151] method binding failure now supports non-Python exception causes --- src/runtime/methodbinder.cs | 8 ++++---- src/runtime/pythonexception.cs | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 6b0976b97..362b1a19c 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -341,13 +341,13 @@ public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int private readonly struct MismatchedMethod { - public MismatchedMethod(PythonException exception, MethodBase mb) + public MismatchedMethod(Exception exception, MethodBase mb) { Exception = exception; Method = mb; } - public PythonException Exception { get; } + public Exception Exception { get; } public MethodBase Method { get; } } @@ -438,8 +438,8 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth outs: out outs); if (margs == null) { - mismatchedMethods.Add(new MismatchedMethod(PythonException.FetchCurrentRaw(), mi)); - Exceptions.Clear(); + var mismatchCause = PythonException.FetchCurrent(); + mismatchedMethods.Add(new MismatchedMethod(mismatchCause, mi)); continue; } if (isOperator) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 3f4bd8154..4e594242c 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -94,6 +94,10 @@ internal static PythonException FetchCurrentRaw() } } + internal static Exception FetchCurrent() + => FetchCurrentOrNull(out _) + ?? throw new InvalidOperationException("No exception is set"); + private static ExceptionDispatchInfo? TryGetDispatchInfo(BorrowedReference exception) { if (exception.IsNull) return null; From d068f365b93903858cddd928a143b20564e0c00a Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 01:05:26 -0700 Subject: [PATCH 098/151] XDecref now checks, that refcount is positive in debug builds --- src/runtime/runtime.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9f6b92cac..3929494d7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -714,6 +714,9 @@ internal static IntPtr SelfIncRef(IntPtr op) internal static unsafe void XDecref(IntPtr op) { +#if DEBUG + Debug.Assert(op == IntPtr.Zero || Refcount(op) > 0); +#endif #if !CUSTOM_INCDEC_REF Py_DecRef(op); return; From 4877fe7d3f52c1dd1cdfdc7e79b263317ee57c13 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 03:09:30 -0700 Subject: [PATCH 099/151] fixed __cause__ on overload bind failure and array conversion --- src/runtime/converter.cs | 5 +++++ src/runtime/methodbinder.cs | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 645f31daa..1425c03e1 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -817,9 +817,14 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo private static void SetConversionError(IntPtr value, Type target) { + // PyObject_Repr might clear the error + Runtime.PyErr_Fetch(out var causeType, out var causeVal, out var causeTrace); + IntPtr ob = Runtime.PyObject_Repr(value); string src = Runtime.GetManagedString(ob); Runtime.XDecref(ob); + + Runtime.PyErr_Restore(causeType.StealNullable(), causeVal.StealNullable(), causeTrace.StealNullable()); Exceptions.RaiseTypeError($"Cannot convert {src} to {target}"); } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 362b1a19c..b2f844900 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -926,7 +926,9 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i } value.Append(": "); + Runtime.PyErr_Fetch(out var errType, out var errVal, out var errTrace); AppendArgumentTypes(to: value, args); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), errTrace.StealNullable()); Exceptions.RaiseTypeError(value.ToString()); return IntPtr.Zero; } From ed594c1cf4920428671f128b81498df3b24907b3 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 12:35:37 -0700 Subject: [PATCH 100/151] cache PythonException message --- src/runtime/pythonexception.cs | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 4e594242c..13316aef9 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -14,16 +14,19 @@ namespace Python.Runtime /// public class PythonException : System.Exception { - public PythonException(PyType type, PyObject? value, PyObject? traceback, - Exception? innerException) - : base("An exception has occurred in Python code. See Message property for details.", innerException) + string message, Exception? innerException) + : base(message, innerException) { Type = type ?? throw new ArgumentNullException(nameof(type)); Value = value; Traceback = traceback; } + public PythonException(PyType type, PyObject? value, PyObject? traceback, + Exception? innerException) + : this(type, value, traceback, GetMessage(value, type), innerException) { } + public PythonException(PyType type, PyObject? value, PyObject? traceback) : this(type, value, traceback, innerException: null) { } @@ -178,7 +181,8 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference private static string GetMessage(PyObject? value, PyType type) { - using var _ = new Py.GILState(); + if (type is null) throw new ArgumentNullException(nameof(type)); + if (value != null && !value.IsNone()) { return value.ToString(); @@ -259,17 +263,6 @@ public override string StackTrace } } - public override string Message - { - get - { - if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) - return "Python error message is unavailable as runtime was shut down"; - - return GetMessage(this.Value, this.Type); - } - } - public bool IsNormalized { get @@ -355,7 +348,8 @@ public string Format() } public PythonException Clone() - => new PythonException(Type, Value, Traceback, InnerException); + => new PythonException(type: Type, value: Value, traceback: Traceback, + Message, InnerException); internal bool Is(IntPtr type) { From e5bce06b4819abfd77542215362a5c26cdb9b26e Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 12:37:27 -0700 Subject: [PATCH 101/151] minor code cleanup --- src/runtime/clrobject.cs | 5 ++++- src/runtime/exceptions.cs | 25 ++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0aa829ee6..2d402596a 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -33,7 +33,7 @@ internal CLRObject(object ob, IntPtr tp) // Fix the BaseException args (and __cause__ in case of Python 3) // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + Exceptions.SetArgsAndCause(ObjectReference); } protected CLRObject() @@ -78,6 +78,9 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } + internal static NewReference GetReference(object ob) + => NewReference.DangerousFromPointer(GetInstHandle(ob)); + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) { CLRObject co = new CLRObject() diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 6a3f2ec6c..02a320c71 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -24,19 +24,10 @@ internal ExceptionClassObject(Type tp) : base(tp) { } - internal static Exception ToException(IntPtr ob) + internal static Exception ToException(BorrowedReference ob) { var co = GetManagedObject(ob) as CLRObject; - if (co == null) - { - return null; - } - var e = co.inst as Exception; - if (e == null) - { - return null; - } - return e; + return co?.inst as Exception; } /// @@ -44,7 +35,7 @@ internal static Exception ToException(IntPtr ob) /// public new static IntPtr tp_repr(IntPtr ob) { - Exception e = ToException(ob); + Exception e = ToException(new BorrowedReference(ob)); if (e == null) { return Exceptions.RaiseTypeError("invalid object"); @@ -67,7 +58,7 @@ internal static Exception ToException(IntPtr ob) /// public new static IntPtr tp_str(IntPtr ob) { - Exception e = ToException(ob); + Exception e = ToException(new BorrowedReference(ob)); if (e == null) { return Exceptions.RaiseTypeError("invalid object"); @@ -157,7 +148,7 @@ internal static void Shutdown() /// pointer. /// /// The python object wrapping - internal static void SetArgsAndCause(IntPtr ob) + internal static void SetArgsAndCause(BorrowedReference ob) { // e: A CLR Exception Exception e = ExceptionClassObject.ToException(ob); @@ -178,13 +169,13 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); + Marshal.WriteIntPtr(ob.DangerousGetAddress(), ExceptionOffset.args, args); if (e.InnerException != null) { // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + using var cause = CLRObject.GetReference(e.InnerException); + Runtime.PyException_SetCause(ob, cause.Steal()); } } From 6819e7b5ea8881c048450079630d79e28261782a Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 12:57:00 -0700 Subject: [PATCH 102/151] improved handling of dict offset in object instances --- src/runtime/interop.cs | 5 ++++- src/runtime/managedtype.cs | 8 ++++++-- src/runtime/typemanager.cs | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index c5958e0f7..cf6345b30 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -185,7 +185,10 @@ public static int magic(IntPtr type) public static int TypeDictOffset(IntPtr type) { - return ManagedDataOffsets.DictOffset(type); + Debug.Assert(TypeOffset.tp_dictoffset > 0); + int dictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(dictOffset > 0); + return dictOffset; } public static int Size(IntPtr pyType) diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index a534e5d99..3cd920ea9 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -260,13 +260,17 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { IntPtr type = Runtime.PyObject_TYPE(ob); - return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + int dictOffset = ObjectOffset.TypeDictOffset(type); + if (dictOffset == 0) return IntPtr.Zero; + return Marshal.ReadIntPtr(ob, dictOffset); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { IntPtr type = Runtime.PyObject_TYPE(ob); - Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + int dictOffset = ObjectOffset.TypeDictOffset(type); + Debug.Assert(dictOffset > 0); + Marshal.WriteIntPtr(ob, dictOffset, value); } } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index fde8fb719..c899b60e2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -165,7 +165,7 @@ internal static IntPtr CreateType(Type impl) // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); + var offset = (IntPtr)ManagedDataOffsets.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); SlotsHolder slotsHolder = CreateSolotsHolder(type); From 6679d1ce5924b93162cd44c8a7b568ea999f89a1 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 12:58:24 -0700 Subject: [PATCH 103/151] added a few debug checks --- src/runtime/runtime.cs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 3929494d7..383de3c30 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1128,13 +1128,31 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); - internal static IntPtr PyObject_Repr(IntPtr pointer) => Delegates.PyObject_Repr(pointer); + internal static IntPtr PyObject_Repr(IntPtr pointer) + { + AssertNoErorSet(); + return Delegates.PyObject_Repr(pointer); + } internal static IntPtr PyObject_Str(IntPtr pointer) => Delegates.PyObject_Str(pointer); - internal static IntPtr PyObject_Unicode(IntPtr pointer) => Delegates.PyObject_Unicode(pointer); + internal static IntPtr PyObject_Unicode(IntPtr pointer) + { + AssertNoErorSet(); + + return Delegates.PyObject_Unicode(pointer); + } + + [Conditional("DEBUG")] + internal static void AssertNoErorSet() + { + if (Exceptions.ErrorOccurred()) + throw new InvalidOperationException( + "Can't call with exception set", + PythonException.FetchCurrent()); + } internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); From dff75d20929da786a0a1d1e61302b68d0a614420 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 14:07:36 -0700 Subject: [PATCH 104/151] do not call find_libpython as Python.Runtime can find it on its own --- pythonnet/__init__.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 692fd5700..188980b8b 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -29,7 +29,6 @@ def load(): if _LOADED: return - from .find_libpython import linked_libpython from os.path import join, dirname if _RUNTIME is None: @@ -38,21 +37,11 @@ def load(): set_default_runtime() dll_path = join(dirname(__file__), "runtime", "Python.Runtime.dll") - libpython = linked_libpython() - - if libpython and _FFI is None and sys.platform != "win32": - # Load and leak libpython handle s.t. the .NET runtime doesn't dlcloses - # it - import posix - - import cffi - _FFI = cffi.FFI() - _FFI.dlopen(libpython, posix.RTLD_NODELETE | posix.RTLD_LOCAL) - + _LOADER_ASSEMBLY = _RUNTIME.get_assembly(dll_path) func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Initialize"] - if func(f"{libpython or ''}".encode("utf8")) != 0: + if func(''.encode("utf8")) != 0: raise RuntimeError("Failed to initialize Python.Runtime.dll") import atexit From 16f04e9c0281cd1e4c266399b35ee89069eb36fe Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 11 Apr 2021 03:09:30 -0700 Subject: [PATCH 105/151] fixed __cause__ on overload bind failure and array conversion also: added debug check --- src/runtime/converter.cs | 5 +++++ src/runtime/finalizer.cs | 2 +- src/runtime/methodbinder.cs | 4 +++- src/runtime/pyobject.cs | 2 +- src/runtime/runtime.cs | 20 +++++++++++--------- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index cd9477a62..6b2e0f648 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -814,9 +814,14 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo private static void SetConversionError(IntPtr value, Type target) { + // PyObject_Repr might clear the error + Runtime.PyErr_Fetch(out var causeType, out var causeVal, out var causeTrace); + IntPtr ob = Runtime.PyObject_Repr(value); string src = Runtime.GetManagedString(ob); Runtime.XDecref(ob); + + Runtime.PyErr_Restore(causeType, causeVal, causeTrace); Exceptions.RaiseTypeError($"Cannot convert {src} to {target}"); } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index fe2e46aac..be4466791 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -54,7 +54,7 @@ public class IncorrectRefCountException : Exception public IncorrectRefCountException(IntPtr ptr) { PyPtr = ptr; - IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + IntPtr pyname = Runtime.PyObject_Str(PyPtr); string name = Runtime.GetManagedString(pyname); Runtime.XDecref(pyname); _message = $"<{name}> may has a incorrect ref count"; diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 8f74e0052..d9572051c 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -876,7 +876,7 @@ protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) { try { - var description = Runtime.PyObject_Unicode(type); + var description = Runtime.PyObject_Str(type); if (description != IntPtr.Zero) { to.Append(Runtime.GetManagedString(description)); @@ -926,7 +926,9 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i } value.Append(": "); + Runtime.PyErr_Fetch(out var errType, out var errVal, out var errTrace); AppendArgumentTypes(to: value, args); + Runtime.PyErr_Restore(errType, errVal, errTrace); Exceptions.RaiseTypeError(value.ToString()); return IntPtr.Zero; } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 81578a7a8..7a1517102 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1040,7 +1040,7 @@ public string Repr() /// public override string ToString() { - IntPtr strval = Runtime.PyObject_Unicode(obj); + IntPtr strval = Runtime.PyObject_Str(obj); string result = Runtime.GetManagedString(strval); Runtime.XDecref(strval); return result; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 263b4473e..4a8d01dd8 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,8 +1,7 @@ -using System.Reflection.Emit; using System; +using System.Diagnostics; using System.Diagnostics.Contracts; using System.Runtime.InteropServices; -using System.Security; using System.Text; using System.Threading; using System.Collections.Generic; @@ -1127,13 +1126,18 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); - internal static IntPtr PyObject_Repr(IntPtr pointer) => Delegates.PyObject_Repr(pointer); - - - internal static IntPtr PyObject_Str(IntPtr pointer) => Delegates.PyObject_Str(pointer); + internal static IntPtr PyObject_Repr(IntPtr pointer) + { + Debug.Assert(PyErr_Occurred() == IntPtr.Zero); + return Delegates.PyObject_Repr(pointer); + } - internal static IntPtr PyObject_Unicode(IntPtr pointer) => Delegates.PyObject_Unicode(pointer); + internal static IntPtr PyObject_Str(IntPtr pointer) + { + Debug.Assert(PyErr_Occurred() == IntPtr.Zero); + return Delegates.PyObject_Str(pointer); + } internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); @@ -2322,7 +2326,6 @@ static Delegates() PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); - PyObject_Unicode = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Str", GetUnmanagedDll(_PythonDll)); PyObject_Dir = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll)); PyObject_GetBuffer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll)); PyBuffer_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll)); @@ -2607,7 +2610,6 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } - internal static delegate* unmanaged[Cdecl] PyObject_Unicode { get; } internal static delegate* unmanaged[Cdecl] PyObject_Dir { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetBuffer { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } From daccc43c326a4b5a63752b01f5eedde3e7da3dca Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 12 Apr 2021 03:30:04 -0700 Subject: [PATCH 106/151] Replaced magic offsets with per-type calculation removed some macro-like method copy-pastes from CPython and bits of dead code All Python types created to represent CLR concepts derive from `CLR MetaType` (as before), which now has two new fields: - `tp_clr_inst_offset`, which is similar to `tp_dictoffset` in that it tells where to find `GCHandle` in instances of this type (e.g. where to find `GCHandle` pointing to `System.Uri` in corresponding Python object) - `tp_clr_inst`, which holds an optional instance of `ManagedType`, that implements the behavior of the type itself (e.g. `GCHandle` pointing to `ClassObject(type = System.Uri)`) So the layout of all Python types created by Python.NET is PyType type; nint tp_clr_inst_offset; GCHandel tp_clr_inst; (points, for example, to an instance of `ClassObject`) When a Python type, that reflects CLR type is created, the layout of instances will be: BaseTypeFields base; optional (if not in base): IntPtr dict; optional (if not in base): IntPtr weaklist; GCHandle gcHandle; (points to `CLRObject` instance, which in turn, for example, points to the actual instance of `System.Uri`) The offset to GC handle is recorded in the Python type's `tp_clr_inst_offset`, and can be found as `PyObject_Type(inst).tp_clr_inst_offset`. Or, preferably, accessor functions in `ManagedType` should be used. --- src/embed_tests/TestPyType.cs | 3 +- src/runtime/classbase.cs | 17 +- src/runtime/classderived.cs | 12 +- src/runtime/clrobject.cs | 23 +-- src/runtime/converter.cs | 6 +- src/runtime/exceptions.cs | 16 +- src/runtime/extensiontype.cs | 6 +- src/runtime/importhook.cs | 28 +-- src/runtime/interop.cs | 272 +---------------------------- src/runtime/managedtype.cs | 108 +++++++++--- src/runtime/metatype.cs | 27 ++- src/runtime/moduleobject.cs | 8 +- src/runtime/native/ABI.cs | 3 - src/runtime/native/ITypeOffsets.cs | 1 + src/runtime/native/TypeOffset.cs | 10 +- src/runtime/runtime.cs | 29 +-- src/runtime/runtime_data.cs | 2 +- src/runtime/typemanager.cs | 195 ++++++++++----------- 18 files changed, 264 insertions(+), 502 deletions(-) diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 02142b782..f70a54c99 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -1,3 +1,4 @@ +using System.Runtime.InteropServices; using System.Text; using NUnit.Framework; @@ -30,7 +31,7 @@ public void CanCreateHeapType() using var doc = new StrPtr(docStr, Encoding.UTF8); var spec = new TypeSpec( name: name, - basicSize: ObjectOffset.Size(Runtime.Runtime.PyTypeType), + basicSize: Marshal.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), slots: new TypeSpec.Slot[] { new (TypeSlotID.tp_doc, doc.RawPointer), }, diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 8b96a96da..bf6a8034d 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,9 +1,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Runtime.Serialization; namespace Python.Runtime { @@ -355,19 +352,21 @@ public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); tp_clear(ob); - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); - self.FreeGCHandle(); + Runtime.PyObject_GC_UnTrack(ob); + Runtime.PyObject_GC_Del(ob); + self?.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { ManagedType self = GetManagedObject(ob); - if (!self.IsTypeObject()) + + bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType; + if (!isTypeObject) { ClearObjectDict(ob); } - self.tpHandle = IntPtr.Zero; + if (self is not null) self.tpHandle = IntPtr.Zero; return 0; } @@ -391,7 +390,7 @@ protected override void OnLoad(InterDomainContext context) SetObjectDict(pyHandle, dict); } gcHandle = AllocGCHandle(); - Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + SetGCHandle(ObjectReference, gcHandle); } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 4e8e88bf3..8b15213c3 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Reflection.Emit; @@ -75,8 +76,8 @@ internal ClassDerivedObject(Type tp) : base(tp) // So we don't call PyObject_GC_Del here and instead we set the python // reference to a weak reference so that the C# object can be collected. GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - int gcOffset = ObjectOffset.magic(Runtime.PyObject_TYPE(self.pyHandle)); - Marshal.WriteIntPtr(self.pyHandle, gcOffset, (IntPtr)gc); + Debug.Assert(self.TypeReference == Runtime.PyObject_TYPE(self.ObjectReference)); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; } @@ -106,7 +107,7 @@ internal static IntPtr ToPython(IPythonDerivedType obj) Runtime._Py_NewReference(self.pyHandle); #endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); - Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; @@ -883,11 +884,6 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); - if (dict != IntPtr.Zero) - { - Runtime.XDecref(dict); - } Runtime.PyObject_GC_Del(self.pyHandle); self.gcHandle.Free(); } diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0aa829ee6..46cd896e2 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,26 +14,16 @@ internal CLRObject(object ob, IntPtr tp) System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Subclass) != 0) - { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); - if (dict == IntPtr.Zero) - { - dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); - } - } - - GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; inst = ob; + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + InitGCHandle(ObjectReference, type: TypeReference, gc); + // Fix the BaseException args (and __cause__ in case of Python 3) // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + if (ob is Exception e) Exceptions.SetArgsAndCause(e, py); } protected CLRObject() @@ -78,6 +68,9 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } + internal static NewReference GetReference(object ob) + => NewReference.DangerousFromPointer(GetInstHandle(ob)); + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) { CLRObject co = new CLRObject() @@ -101,7 +94,7 @@ protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + SetGCHandle(ObjectReference, TypeReference, gc); } } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 6b2e0f648..2e10e9041 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -580,7 +580,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } @@ -606,7 +606,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } @@ -632,7 +632,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 06d2d55b5..40e018fe6 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -156,15 +156,8 @@ internal static void Shutdown() /// pointer. /// /// The python object wrapping - internal static void SetArgsAndCause(IntPtr ob) + internal static void SetArgsAndCause(Exception e, IntPtr ob) { - // e: A CLR Exception - Exception e = ExceptionClassObject.ToException(ob); - if (e == null) - { - return; - } - IntPtr args; if (!string.IsNullOrEmpty(e.Message)) { @@ -177,13 +170,14 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); + if (Runtime.PyObject_SetAttrString(ob, "args", args) != 0) + throw new PythonException(); if (e.InnerException != null) { // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + using var cause = CLRObject.GetReference(e.InnerException); + Runtime.PyException_SetCause(ob, cause.DangerousMoveToPointer()); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index a5f0f1219..554837c46 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -33,13 +33,17 @@ public ExtensionType() tpHandle = tp; pyHandle = py; +#if DEBUG + GetGCHandle(ObjectReference, TypeReference, out var existing); + System.Diagnostics.Debug.Assert(existing == IntPtr.Zero); +#endif SetupGc(); } void SetupGc () { GCHandle gc = AllocGCHandle(TrackTypes.Extension); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + InitGCHandle(ObjectReference, TypeReference, gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 184b588ad..be2281c8f 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -15,26 +15,6 @@ internal static class ImportHook private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); - private static IntPtr module_def = IntPtr.Zero; - - internal static void InitializeModuleDef() - { - if (module_def == IntPtr.Zero) - { - module_def = ModuleDefOffset.AllocModuleDef("clr"); - } - } - - internal static void ReleaseModuleDef() - { - if (module_def == IntPtr.Zero) - { - return; - } - ModuleDefOffset.FreeModuleDef(module_def); - module_def = IntPtr.Zero; - } - /// /// Initialize just the __import__ hook itself. /// @@ -90,8 +70,7 @@ internal static unsafe void Initialize() root = new CLRModule(); // create a python module with the same methods as the clr module-like object - InitializeModuleDef(); - py_clr_module = Runtime.PyModule_Create2(module_def, 3); + py_clr_module = Runtime.PyModule_New("clr").DangerousMoveToPointer(); // both dicts are borrowed references BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); @@ -116,13 +95,8 @@ internal static void Shutdown() RestoreImport(); - bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; - if (shouldFreeDef) - { - ReleaseModuleDef(); - } Runtime.XDecref(root.pyHandle); root = null; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index c5958e0f7..188db3a58 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -68,276 +68,6 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - public static int magic() => ManagedDataOffsets.Magic; - } - - internal static class ManagedDataOffsets - { - public static int Magic { get; internal set; } - public static readonly Dictionary NameMapping = new Dictionary(); - - static class DataOffsets - { - public static readonly int ob_data = 0; - public static readonly int ob_dict = 0; - - static DataOffsets() - { - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fields.Length; i++) - { - fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); - } - } - } - - static ManagedDataOffsets() - { - NameMapping = TypeOffset.GetOffsets(); - - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - size = fields.Length * IntPtr.Size; - } - - public static int GetSlotOffset(string name) - { - return NameMapping[name]; - } - - private static int BaseOffset(IntPtr type) - { - Debug.Assert(type != IntPtr.Zero); - int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0); - return typeSize; - } - - public static int DataOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_data; - } - - public static int DictOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_dict; - } - - public static int ob_data => DataOffsets.ob_data; - public static int ob_dict => DataOffsets.ob_dict; - public static int Size { get { return size; } } - - private static readonly int size; - } - - internal static class OriginalObjectOffsets - { - static OriginalObjectOffsets() - { - int size = IntPtr.Size; - var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD -#if PYTHON_WITH_PYDEBUG - _ob_next = 0; - _ob_prev = 1 * size; - n = 2; -#endif - ob_refcnt = (n + 0) * size; - ob_type = (n + 1) * size; - } - - public static int Size { get { return size; } } - - private static readonly int size = -#if PYTHON_WITH_PYDEBUG - 4 * IntPtr.Size; -#else - 2 * IntPtr.Size; -#endif - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset - { - static ObjectOffset() - { -#if PYTHON_WITH_PYDEBUG - _ob_next = OriginalObjectOffsets._ob_next; - _ob_prev = OriginalObjectOffsets._ob_prev; -#endif - ob_refcnt = OriginalObjectOffsets.ob_refcnt; - ob_type = OriginalObjectOffsets.ob_type; - - size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int magic(IntPtr type) - { - return ManagedDataOffsets.DataOffset(type); - } - - public static int TypeDictOffset(IntPtr type) - { - return ManagedDataOffsets.DictOffset(type); - } - - public static int Size(IntPtr pyType) - { - if (IsException(pyType)) - { - return ExceptionOffset.Size(); - } - - return size; - } - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - private static readonly int size; - - private static bool IsException(IntPtr pyObjectPtr) - { - var pyObject = new BorrowedReference(pyObjectPtr); - var type = Runtime.PyObject_TYPE(pyObject); - return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) - || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - } - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ExceptionOffset - { - static ExceptionOffset() - { - Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); - } - - size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int Size() { return size; } - - // PyException_HEAD - // (start after PyObject_HEAD) - public static int dict = 0; - public static int args = 0; - public static int traceback = 0; - public static int context = 0; - public static int cause = 0; - public static int suppress_context = 0; - - private static readonly int size; - } - - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class BytesOffset - { - static BytesOffset() - { - Type type = typeof(BytesOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - /* The *real* layout of a type object when allocated on the heap */ - //typedef struct _heaptypeobject { -#if PYTHON_WITH_PYDEBUG -/* _PyObject_HEAD_EXTRA defines pointers to support a doubly-linked list of all live heap objects. */ - public static int _ob_next = 0; - public static int _ob_prev = 0; -#endif - // PyObject_VAR_HEAD { - // PyObject_HEAD { - public static int ob_refcnt = 0; - public static int ob_type = 0; - // } - public static int ob_size = 0; /* Number of items in _VAR_iable part */ - // } - public static int ob_shash = 0; - public static int ob_sval = 0; /* start of data */ - - /* Invariants: - * ob_sval contains space for 'ob_size+1' elements. - * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. - */ - //} PyBytesObject; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ModuleDefOffset - { - static ModuleDefOffset() - { - Type type = typeof(ModuleDefOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * size) + TypeOffset.ob_size); - } - } - - public static IntPtr AllocModuleDef(string modulename) - { - byte[] ascii = Encoding.ASCII.GetBytes(modulename); - int size = name + ascii.Length + 1; - IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i <= m_free; i += IntPtr.Size) - Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); - Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); - Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); - Marshal.WriteByte(ptr, name + ascii.Length, 0); - return ptr; - } - - public static void FreeModuleDef(IntPtr ptr) - { - Marshal.FreeHGlobal(ptr); - } - - // typedef struct PyModuleDef{ - // typedef struct PyModuleDef_Base { - // starts after PyObject_HEAD (TypeOffset.ob_type + 1) - public static int m_init = 0; - public static int m_index = 0; - public static int m_copy = 0; - // } PyModuleDef_Base - public static int m_name = 0; - public static int m_doc = 0; - public static int m_size = 0; - public static int m_methods = 0; - public static int m_reload = 0; - public static int m_traverse = 0; - public static int m_clear = 0; - public static int m_free = 0; - // } PyModuleDef - - public static int name = 0; - } - - /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -357,7 +87,7 @@ public enum TypeFlags: int HaveStacklessExtension = 0, /* XXX Reusing reserved constants */ /// PythonNet specific - Managed = (1 << 15), + HasClrInstance = (1 << 15), /// PythonNet specific Subclass = (1 << 16), HaveIndex = (1 << 17), diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index d3ee697fd..41408abc7 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -28,6 +28,7 @@ internal enum TrackTypes internal IntPtr tpHandle; // PyType * internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); + internal BorrowedReference TypeReference => new BorrowedReference(tpHandle); private static readonly Dictionary _managedObjs = new Dictionary(); @@ -93,17 +94,10 @@ internal static ManagedType GetManagedObject(IntPtr ob) } var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + if ((flags & TypeFlags.HasClrInstance) != 0) { - IntPtr op = tp == ob - ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); - if (op == IntPtr.Zero) - { - return null; - } - var gc = (GCHandle)op; - return (ManagedType)gc.Target; + var gc = TryGetGCHandle(new BorrowedReference(ob)); + return (ManagedType)gc?.Target; } } return null; @@ -118,10 +112,9 @@ internal static ManagedType GetManagedObjectType(IntPtr ob) { IntPtr tp = Runtime.PyObject_TYPE(ob); var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + if ((flags & TypeFlags.HasClrInstance) != 0) { - tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - var gc = (GCHandle)tp; + var gc = GetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType); return (ManagedType)gc.Target; } } @@ -140,9 +133,9 @@ internal static ManagedType GetManagedObjectErr(IntPtr ob) } - internal static bool IsManagedType(BorrowedReference ob) - => IsManagedType(ob.DangerousGetAddressOrNull()); - internal static bool IsManagedType(IntPtr ob) + internal static bool IsInstanceOfManagedType(BorrowedReference ob) + => IsInstanceOfManagedType(ob.DangerousGetAddressOrNull()); + internal static bool IsInstanceOfManagedType(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -152,18 +145,15 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - return true; - } + return IsManagedType(new BorrowedReference(tp)); } return false; } - public bool IsTypeObject() + internal static bool IsManagedType(BorrowedReference type) { - return pyHandle == tpHandle; + var flags = (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags); + return (flags & TypeFlags.HasClrInstance) != 0; } internal static IDictionary GetManagedObjects() @@ -256,13 +246,81 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { IntPtr type = Runtime.PyObject_TYPE(ob); - return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + return Marshal.ReadIntPtr(ob, instanceDictOffset); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { IntPtr type = Runtime.PyObject_TYPE(ob); - Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + Marshal.WriteIntPtr(ob, instanceDictOffset, value); + } + + internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) + { + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || type == Runtime.CLRMetaType); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int gcHandleOffset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(gcHandleOffset > 0); + + handle = Marshal.ReadIntPtr(reflectedClrObject.DangerousGetAddress(), gcHandleOffset); + } + + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + GetGCHandle(reflectedClrObject, type, out IntPtr handle); + return handle == IntPtr.Zero ? null : (GCHandle)handle; + } + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject) + { + BorrowedReference reflectedType = Runtime.PyObject_TYPE(reflectedClrObject); + + return TryGetGCHandle(reflectedClrObject, reflectedType); + } + + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject) + => TryGetGCHandle(reflectedClrObject) ?? throw new InvalidOperationException(); + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + => TryGetGCHandle(reflectedClrObject, type) ?? throw new InvalidOperationException(); + + internal static void InitGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle handle) + { + Debug.Assert(TryGetGCHandle(reflectedClrObject) == null); + + SetGCHandle(reflectedClrObject, type: type, handle); + } + internal static void InitGCHandle(BorrowedReference reflectedClrObject, GCHandle handle) + => InitGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), handle); + + internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle newHandle) + { + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + Marshal.WriteIntPtr(reflectedClrObject.DangerousGetAddress(), offset, (IntPtr)newHandle); + } + internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) + => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); + + internal static class Offsets + { + static Offsets() + { + int pyTypeSize = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize); + if (pyTypeSize < 0) throw new InvalidOperationException(); + + tp_clr_inst_offset = pyTypeSize; + tp_clr_inst = tp_clr_inst_offset + IntPtr.Size; + } + public static int tp_clr_inst_offset { get; } + public static int tp_clr_inst { get; } } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 68dae2508..1fde7dd78 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,6 +1,4 @@ using System; -using System.Collections; -using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -148,7 +146,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) } var flags = TypeFlags.Default; - flags |= TypeFlags.Managed; + flags |= TypeFlags.HasClrInstance; flags |= TypeFlags.HeapType; flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; @@ -164,10 +162,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); + // derived types must have their GCHandle at the same offset as the base types + int clrInstOffset = Marshal.ReadInt32(base_type, Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, Offsets.tp_clr_inst_offset, clrInstOffset); // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); + IntPtr gc = Marshal.ReadIntPtr(base_type, Offsets.tp_clr_inst); + Marshal.WriteIntPtr(type, Offsets.tp_clr_inst, gc); + + if (Runtime.PyType_Ready(type) != 0) + throw new PythonException(); return type; } @@ -205,6 +209,11 @@ public static IntPtr tp_call(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } + return CallInit(obj, args, kw); + } + + private static IntPtr CallInit(IntPtr obj, IntPtr args, IntPtr kw) + { var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); Runtime.PyErr_Clear(); @@ -288,8 +297,12 @@ public static void tp_dealloc(IntPtr tp) var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { - IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - ((GCHandle)gc).Free(); + GetGCHandle(new BorrowedReference(tp)).Free(); +#if DEBUG + // prevent ExecutionEngineException in debug builds in case we have a bug + // this would allow using managed debugger to investigate the issue + SetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType, default); +#endif } IntPtr op = Marshal.ReadIntPtr(tp, TypeOffset.ob_type); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 41167e322..3c4e02a23 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -44,7 +44,10 @@ public ModuleObject(string name) docstring += "- " + a.FullName + "\n"; } - dict = Runtime.PyDict_New(); + var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); + PythonException.ThrowIfIsNull(dictRef); + dict = dictRef.DangerousMoveToPointer(); + using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); @@ -54,9 +57,6 @@ public ModuleObject(string name) Runtime.PyDict_SetItem(DictRef, PyIdentifier.__doc__, pydocstring); Runtime.PyDict_SetItem(DictRef, PyIdentifier.__class__, pycls); - Runtime.XIncref(dict); - SetObjectDict(pyHandle, dict); - InitializeModuleMembers(); } diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index 3264531de..339919dee 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -4,7 +4,6 @@ namespace Python.Runtime.Native using System.Globalization; using System.Linq; using System.Reflection; - using System.Runtime.InteropServices; static class ABI { @@ -29,8 +28,6 @@ internal static void Initialize(Version version, BorrowedReference pyType) } var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); TypeOffset.Use(typeOffsets); - - ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); } } } diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 485c041f8..0829e5bc9 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -63,6 +63,7 @@ interface ITypeOffsets int tp_new { get; } int tp_repr { get; } int tp_richcompare { get; } + int tp_weaklistoffset { get; } int tp_setattro { get; } int tp_str { get; } int tp_traverse { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 9f5ed671b..edbbe3b2c 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -70,6 +70,7 @@ static partial class TypeOffset internal static int tp_new { get; private set; } internal static int tp_repr { get; private set; } internal static int tp_richcompare { get; private set; } + internal static int tp_weaklistoffset { get; private set; } internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } @@ -91,9 +92,13 @@ internal static void Use(ITypeOffsets offsets) ValidateUnusedTypeOffsetProperties(offsetProperties); ValidateRequiredOffsetsPresent(offsetProperties); + + SlotOffsets = GetOffsets(); } static readonly BindingFlags FieldFlags = BindingFlags.NonPublic | BindingFlags.Static; + + static Dictionary SlotOffsets; internal static Dictionary GetOffsets() { var properties = typeof(TypeOffset).GetProperties(FieldFlags); @@ -104,10 +109,9 @@ internal static Dictionary GetOffsets() return result; } - internal static int GetOffsetUncached(string name) + public static int GetSlotOffset(string slotName) { - var property = typeof(TypeOffset).GetProperty(name, FieldFlags); - return (int)property.GetValue(obj: null, index: null); + return SlotOffsets[slotName]; } static readonly HashSet slotNames = new HashSet(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4a8d01dd8..1b1a7eccc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -470,7 +470,7 @@ private static void ClearClrModules() var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item, 0); var module = PyTuple_GetItem(item, 1); - if (ManagedType.IsManagedType(module)) + if (ManagedType.IsInstanceOfManagedType(module)) { PyDict_DelItem(modules, name); } @@ -523,6 +523,7 @@ private static void MoveClrInstancesOnwershipToPython() if (obj.gcHandle.IsAllocated) { obj.gcHandle.Free(); + ManagedType.SetGCHandle(obj.ObjectReference, default); } obj.gcHandle = default; } @@ -568,6 +569,8 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; + internal static BorrowedReference CLRMetaType => new BorrowedReference(PyCLRMetaType); + public static PyObject None { get @@ -1000,7 +1003,7 @@ internal static IntPtr PyObject_Type(IntPtr op) internal static string PyObject_GetTypeName(IntPtr op) { - IntPtr pyType = Marshal.ReadIntPtr(op, ObjectOffset.ob_type); + IntPtr pyType = PyObject_TYPE(op); IntPtr ppName = Marshal.ReadIntPtr(pyType, TypeOffset.tp_name); return Marshal.PtrToStringAnsi(ppName); } @@ -1010,7 +1013,7 @@ internal static string PyObject_GetTypeName(IntPtr op) /// internal static bool PyObject_IsIterable(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iter = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iter); return tp_iter != IntPtr.Zero; } @@ -1508,6 +1511,13 @@ internal static IntPtr EmptyPyBytes() return Delegates.PyBytes_FromString((IntPtr)bytes); } + internal static IntPtr PyBytes_AsString(IntPtr ob) => PyBytes_AsString(new BorrowedReference(ob)); + internal static IntPtr PyBytes_AsString(BorrowedReference ob) + { + Debug.Assert(ob != null); + return Delegates.PyBytes_AsString(ob); + } + internal static long PyBytes_Size(IntPtr op) { return (long)_PyBytes_Size(op); @@ -1516,11 +1526,6 @@ internal static long PyBytes_Size(IntPtr op) private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); - internal static IntPtr PyBytes_AS_STRING(IntPtr ob) - { - return ob + BytesOffset.ob_sval; - } - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) { @@ -1615,7 +1620,7 @@ internal static string GetManagedString(IntPtr op) { using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); int length = (int)PyUnicode_GetSize(op); - char* codePoints = (char*)PyBytes_AS_STRING(p.DangerousGetAddress()); + char* codePoints = (char*)PyBytes_AsString(p.DangerousGetAddress()); return new string(codePoints, startIndex: 1, // skip BOM length: length); @@ -1876,7 +1881,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static bool PyIter_Check(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iternext = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iternext); return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } @@ -2248,7 +2253,7 @@ internal static IntPtr GetBuiltins() return PyImport_Import(PyIdentifier.builtins); } - private static class Delegates + internal static class Delegates { static readonly ILibraryLoader libraryLoader = LibraryLoader.Instance; @@ -2406,6 +2411,7 @@ static Delegates() _PySequence_Count = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Count", GetUnmanagedDll(_PythonDll)); PySequence_Tuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Tuple), GetUnmanagedDll(_PythonDll)); PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); + PyBytes_AsString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll)); PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll)); @@ -2683,6 +2689,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] _PySequence_Count { get; } internal static delegate* unmanaged[Cdecl] PySequence_Tuple { get; } internal static delegate* unmanaged[Cdecl] PySequence_List { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_AsString { get; } internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromStringAndSize { get; } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 0b3bf3017..29cea4181 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -279,7 +279,7 @@ private static void SaveRuntimeDataModules(RuntimeDataStorage storage) var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); - if (ManagedType.IsManagedType(module)) + if (ManagedType.IsInstanceOfManagedType(module)) { XIncref(name); XIncref(module); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 01aceb656..20b95d85c 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -134,7 +134,7 @@ internal static BorrowedReference GetTypeReference(Type type) /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. /// - internal static IntPtr GetTypeHandle(ManagedType obj, Type type) + internal static IntPtr GetTypeHandle(ClassBase obj, Type type) { IntPtr handle; cache.TryGetValue(type, out handle); @@ -157,21 +157,28 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// - internal static IntPtr CreateType(Type impl) + internal static unsafe IntPtr CreateType(Type impl) { - IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); - int ob_size = ObjectOffset.Size(type); + IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); + IntPtr base_ = impl == typeof(CLRModule) + ? Runtime.PyModuleType + : Runtime.PyBaseObjectType; + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + int tp_clr_inst_offset = newFieldOffset; + newFieldOffset += IntPtr.Size; + + int ob_size = newFieldOffset; // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); + Marshal.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); - var flags = TypeFlags.Default | TypeFlags.Managed | + var flags = TypeFlags.Default | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); @@ -194,7 +201,7 @@ internal static IntPtr CreateType(Type impl) } - internal static IntPtr CreateType(ManagedType impl, Type clrType) + internal static IntPtr CreateType(ClassBase impl, Type clrType) { // Cleanup the type name to get rid of funny nested type names. string name = $"clr.{clrType.FullName}"; @@ -209,19 +216,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) name = name.Substring(i + 1); } - IntPtr base_ = IntPtr.Zero; - int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - - // XXX Hack, use a different base class for System.Exception - // Python 2.5+ allows new style class exceptions but they *must* - // subclass BaseException (or better Exception). - if (typeof(Exception).IsAssignableFrom(clrType)) - { - ob_size = ObjectOffset.Size(Exceptions.Exception); - } - - int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; - + IntPtr base_ = Runtime.PyBaseObjectType; if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -229,17 +224,32 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) else if (clrType.BaseType != null) { ClassBase bc = ClassManager.GetClass(clrType.BaseType); - base_ = bc.pyHandle; + if (bc.ObjectReference != null) + { + // there are cases when base class has not been fully initialized yet (nested types) + base_ = bc.pyHandle; + } } IntPtr type = AllocateTypeObject(name, Runtime.PyCLRMetaType); - Marshal.WriteIntPtr(type, TypeOffset.ob_type, Runtime.PyCLRMetaType); - Runtime.XIncref(Runtime.PyCLRMetaType); + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + if (ManagedType.IsManagedType(new BorrowedReference(base_))) + { + int baseClrInstOffset = Marshal.ReadInt32(base_, ManagedType.Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); + } + else + { + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); + newFieldOffset += IntPtr.Size; + } + + int ob_size = newFieldOffset; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method SlotsHolder slotsHolder = CreateSolotsHolder(type); @@ -260,24 +270,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // Only set mp_subscript and mp_ass_subscript for types with indexers - if (impl is ClassBase cb) + if (!(impl is ArrayObject)) { - if (!(impl is ArrayObject)) + if (impl.indexer == null || !impl.indexer.CanGet) { - if (cb.indexer == null || !cb.indexer.CanGet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - } - if (cb.indexer == null || !cb.indexer.CanSet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); - } + Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); + } + if (impl.indexer == null || !impl.indexer.CanSet) + { + Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } - } - else - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } if (base_ != IntPtr.Zero) @@ -287,7 +289,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) } const TypeFlags flags = TypeFlags.Default - | TypeFlags.Managed + | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.BaseType | TypeFlags.HaveGC; @@ -309,9 +311,11 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); mod.Dispose(); + var typeRef = new BorrowedReference(type); + // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); - Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); + ManagedType.InitGCHandle(typeRef, Runtime.CLRMetaType, gc); // Set the handle attributes on the implementing instance. impl.tpHandle = type; @@ -322,6 +326,31 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } + static int InheritOrAllocateStandardFields(IntPtr type, IntPtr @base) + { + int baseSize = Marshal.ReadInt32(@base, TypeOffset.tp_basicsize); + int newFieldOffset = baseSize; + + void InheritOrAllocate(int typeField) + { + int value = Marshal.ReadInt32(@base, typeField); + if (value == 0) + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); + newFieldOffset += IntPtr.Size; + } + else + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); + } + } + + InheritOrAllocate(TypeOffset.tp_dictoffset); + InheritOrAllocate(TypeOffset.tp_weaklistoffset); + + return newFieldOffset; + } + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { var dictRef = new BorrowedReference(py_dict); @@ -454,11 +483,14 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - int size = TypeOffset.magic() + IntPtr.Size; + int size = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + + IntPtr.Size // tp_clr_inst + ; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); const TypeFlags flags = TypeFlags.Default - | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); @@ -544,53 +576,6 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) - { - // Utility to create a subtype of a std Python type, but with - // a managed type able to override implementation - - IntPtr type = AllocateTypeObject(name, metatype: Runtime.PyTypeType); - //Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); - //Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - - //IntPtr offset = (IntPtr)ObjectOffset.ob_dict; - //Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - - //IntPtr dc = Runtime.PyDict_Copy(dict); - //Marshal.WriteIntPtr(type, TypeOffset.tp_dict, dc); - - Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); - Runtime.XIncref(base_); - - var flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); - - CopySlot(base_, type, TypeOffset.tp_traverse); - CopySlot(base_, type, TypeOffset.tp_clear); - CopySlot(base_, type, TypeOffset.tp_is_gc); - - SlotsHolder slotsHolder = CreateSolotsHolder(type); - InitializeSlots(type, impl, slotsHolder); - - if (Runtime.PyType_Ready(type) != 0) - { - throw new PythonException(); - } - - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); - - // The type has been modified after PyType_Ready has been called - // Refresh the type - Runtime.PyType_Modified(type); - - return type; - } - /// /// Utility method to allocate a type object & do basic initialization. @@ -598,6 +583,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); + PythonException.ThrowIfIsNull(type); // Clr type would not use __slots__, // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), // thus set the ob_size to 0 for avoiding slots iterations. @@ -670,7 +656,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo { continue; } - var offset = ManagedDataOffsets.GetSlotOffset(slot); + var offset = TypeOffset.GetSlotOffset(slot); Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } } @@ -688,7 +674,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo /// Can override the slot when it existed static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) { - var offset = ManagedDataOffsets.GetSlotOffset(name); + var offset = TypeOffset.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { return; @@ -698,7 +684,7 @@ static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverri static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { - int offset = ManagedDataOffsets.GetSlotOffset(name); + int offset = TypeOffset.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { @@ -723,7 +709,7 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots static bool IsSlotSet(IntPtr type, string name) { - int offset = ManagedDataOffsets.GetSlotOffset(name); + int offset = TypeOffset.GetSlotOffset(name); return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } @@ -794,6 +780,8 @@ class SlotsHolder private List _deallocators = new List(); private bool _alreadyReset = false; + BorrowedReference Type => new BorrowedReference(_type); + /// /// Create slots holder for holding the delegate of slots and be able to reset them. /// @@ -861,15 +849,18 @@ public void ResetSlots() _deallocators.Clear(); // Custom reset - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); - if (handlePtr != IntPtr.Zero) + if (Type != Runtime.CLRMetaType) { - GCHandle handle = GCHandle.FromIntPtr(handlePtr); - if (handle.IsAllocated) + var metatype = Runtime.PyObject_TYPE(Type); + if (ManagedType.TryGetGCHandle(Type, metatype) is { } handle) { - handle.Free(); + if (handle.IsAllocated) + { + handle.Free(); + } + + ManagedType.SetGCHandle(Type, metatype, default); } - Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); } } From db746aa86a2c2b1d7be389e6daaa3e41ff832e20 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 26 Mar 2021 14:19:56 -0700 Subject: [PATCH 107/151] detect Py_TRACE_REFS at runtime and calculate object offsets accordingly --- src/runtime/classderived.cs | 5 +- src/runtime/converter.cs | 10 ++++ src/runtime/native/ABI.cs | 25 +++++++- src/runtime/native/TypeOffset.cs | 3 +- src/runtime/runtime.cs | 99 ++++++++++++++++++-------------- 5 files changed, 91 insertions(+), 51 deletions(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 8b15213c3..f8b97b397 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -102,10 +102,7 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { - -#if PYTHON_WITH_PYDEBUG - Runtime._Py_NewReference(self.pyHandle); -#endif + Runtime._Py_NewReference(self.ObjectReference); GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 2e10e9041..70b3d9eaa 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -525,6 +525,16 @@ internal static bool ToManagedValue(IntPtr value, Type obType, internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + internal static int ToInt32(BorrowedReference value) + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + return checked((int)num); + } + /// /// Convert a Python value to an instance of a primitive managed type. /// diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index 339919dee..e99fc33ab 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -7,6 +7,9 @@ namespace Python.Runtime.Native static class ABI { + public static int RefCountOffset { get; } = GetRefCountOffset(); + public static int ObjectHeadOffset => RefCountOffset; + internal static void Initialize(Version version, BorrowedReference pyType) { string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture, @@ -16,18 +19,34 @@ internal static void Initialize(Version version, BorrowedReference pyType) const string nativeTypeOffsetClassName = "Python.Runtime.NativeTypeOffset"; string className = "Python.Runtime.TypeOffset" + offsetsClassSuffix; + Type nativeOffsetsClass = thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false); Type typeOffsetsClass = // Try platform native offsets first. It is only present when generated by setup.py - thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false) - ?? thisAssembly.GetType(className, throwOnError: false); + nativeOffsetsClass ?? thisAssembly.GetType(className, throwOnError: false); if (typeOffsetsClass is null) { var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset")); string message = $"Searching for {className}, found {string.Join(",", types)}."; throw new NotSupportedException($"Python ABI v{version} is not supported: {message}"); } + var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); - TypeOffset.Use(typeOffsets); + TypeOffset.Use(typeOffsets, nativeOffsetsClass == null ? ObjectHeadOffset : 0); + } + + static unsafe int GetRefCountOffset() + { + IntPtr tempObject = Runtime.PyList_New(0); + IntPtr* tempPtr = (IntPtr*)tempObject; + int offset = 0; + while(tempPtr[offset] != (IntPtr)1) + { + offset++; + if (offset > 100) + throw new InvalidProgramException("PyObject_HEAD could not be found withing reasonable distance from the start of PyObject"); + } + Runtime.XDecref(tempObject); + return offset * IntPtr.Size; } } } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index edbbe3b2c..4e5a726bc 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -75,7 +75,7 @@ static partial class TypeOffset internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } - internal static void Use(ITypeOffsets offsets) + internal static void Use(ITypeOffsets offsets, int extraHeadOffset) { if (offsets is null) throw new ArgumentNullException(nameof(offsets)); @@ -87,6 +87,7 @@ internal static void Use(ITypeOffsets offsets) var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name); int value = (int)sourceProperty.GetValue(offsets, null); + value += extraHeadOffset; offsetProperty.SetValue(obj: null, value: value, index: null); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1b1a7eccc..6ebf885a0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -85,9 +85,9 @@ internal static Version PyVersion { using (var versionTuple = new PyTuple(PySys_GetObject("version_info"))) { - var major = versionTuple[0].As(); - var minor = versionTuple[1].As(); - var micro = versionTuple[2].As(); + var major = Converter.ToInt32(versionTuple[0].Reference); + var minor = Converter.ToInt32(versionTuple[1].Reference); + var micro = Converter.ToInt32(versionTuple[2].Reference); return new Version(major, minor, micro); } } @@ -198,15 +198,15 @@ private static void InitPyMembers() SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), () => PyFalse = IntPtr.Zero); - SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + SetPyMemberTypeOf(ref PyBoolType, PyTrue, () => PyBoolType = IntPtr.Zero); - SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + SetPyMemberTypeOf(ref PyNoneType, PyNone, () => PyNoneType = IntPtr.Zero); - SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + SetPyMemberTypeOf(ref PyTypeType, PyNoneType, () => PyTypeType = IntPtr.Zero); op = PyObject_GetAttrString(builtins, "len"); - SetPyMember(ref PyMethodType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyMethodType, op, () => PyMethodType = IntPtr.Zero); XDecref(op); @@ -215,7 +215,7 @@ private static void InitPyMembers() // // object.__init__ seems safe, though. op = PyObject_GetAttr(PyBaseObjectType, PyIdentifier.__init__); - SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyWrapperDescriptorType, op, () => PyWrapperDescriptorType = IntPtr.Zero); XDecref(op); @@ -226,47 +226,47 @@ private static void InitPyMembers() } op = PyString_FromString("string"); - SetPyMember(ref PyStringType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyStringType, op, () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - SetPyMember(ref PyUnicodeType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyUnicodeType, op, () => PyUnicodeType = IntPtr.Zero); XDecref(op); op = EmptyPyBytes(); - SetPyMember(ref PyBytesType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyBytesType, op, () => PyBytesType = IntPtr.Zero); XDecref(op); op = PyTuple_New(0); - SetPyMember(ref PyTupleType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyTupleType, op, () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - SetPyMember(ref PyListType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyListType, op, () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - SetPyMember(ref PyDictType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyDictType, op, () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - SetPyMember(ref PyIntType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyIntType, op, () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - SetPyMember(ref PyLongType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyLongType, op, () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - SetPyMember(ref PyFloatType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyFloatType, op, () => PyFloatType = IntPtr.Zero); XDecref(op); @@ -278,7 +278,8 @@ private static void InitPyMembers() _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); { using var sys = PyImport_ImportModule("sys"); - PyModuleType = PyObject_Type(sys.DangerousMoveToPointer()); + SetPyMemberTypeOf(ref PyModuleType, sys.DangerousGetAddress(), + () => PyModuleType = IntPtr.Zero); } } @@ -455,6 +456,12 @@ private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) _pyRefs.Add(value, onRelease); } + private static void SetPyMemberTypeOf(ref IntPtr obj, IntPtr value, Action onRelease) + { + var type = PyObject_Type(new BorrowedReference(value)).DangerousMoveToPointer(); + SetPyMember(ref obj, type, onRelease); + } + private static void ResetPyMembers() { _pyRefs.Release(); @@ -761,16 +768,12 @@ internal static unsafe void XDecref(IntPtr op) [Pure] internal static unsafe long Refcount(IntPtr op) { -#if PYTHON_WITH_PYDEBUG - var p = (void*)(op + TypeOffset.ob_refcnt); -#else - var p = (void*)op; -#endif - if ((void*)0 == p) + if (op == IntPtr.Zero) { return 0; } - return Is32Bit ? (*(int*)p) : (*(long*)p); + var p = (nint*)(op + ABI.RefCountOffset); + return *p; } /// @@ -977,14 +980,9 @@ internal static unsafe IntPtr PyObject_TYPE(IntPtr op) { return IntPtr.Zero; } -#if PYTHON_WITH_PYDEBUG - var n = 3; -#else - var n = 1; -#endif - return Is32Bit - ? new IntPtr((void*)(*((uint*)p + n))) - : new IntPtr((void*)(*((ulong*)p + n))); + Debug.Assert(TypeOffset.ob_type > 0); + IntPtr* typePtr = (IntPtr*)(op + TypeOffset.ob_type); + return *typePtr; } internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) => new BorrowedReference(PyObject_TYPE(op.DangerousGetAddress())); @@ -1001,6 +999,9 @@ internal static IntPtr PyObject_Type(IntPtr op) return tp; } + internal static NewReference PyObject_Type(BorrowedReference o) + => Delegates.PyObject_Type(o); + internal static string PyObject_GetTypeName(IntPtr op) { IntPtr pyType = PyObject_TYPE(op); @@ -1145,10 +1146,11 @@ internal static IntPtr PyObject_Str(IntPtr pointer) internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); -#if PYTHON_WITH_PYDEBUG - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void _Py_NewReference(IntPtr ob); -#endif + internal static void _Py_NewReference(BorrowedReference ob) + { + if (Delegates._Py_NewReference != null) + Delegates._Py_NewReference(ob); + } //==================================================================== // Python buffer API @@ -1912,11 +1914,6 @@ internal static string PyModule_GetName(IntPtr module) internal static string PyModule_GetFilename(IntPtr module) => Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8); -#if PYTHON_WITH_PYDEBUG - [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] -#else - -#endif internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver); @@ -2331,6 +2328,7 @@ static Delegates() PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); + PyObject_Type = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Type), GetUnmanagedDll(_PythonDll)); PyObject_Dir = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll)); PyObject_GetBuffer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll)); PyBuffer_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll)); @@ -2466,7 +2464,14 @@ static Delegates() PyModule_GetName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll)); PyModule_GetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll)); PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); - PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); + try + { + PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); + } PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2521,6 +2526,12 @@ static Delegates() PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); + + try + { + _Py_NewReference = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_Py_NewReference), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) { } } static global::System.IntPtr GetUnmanagedDll(string libraryName) @@ -2616,6 +2627,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Type { get; } internal static delegate* unmanaged[Cdecl] PyObject_Dir { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetBuffer { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } @@ -2799,6 +2811,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } + internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; } } } From 33d771c2804d82a9c4cf740a4564338d32e37e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 3 May 2021 03:08:43 -0400 Subject: [PATCH 108/151] Re-enable the domain reload tests (#1404) Add `domain_tests` to `testpaths` in pyproject.toml Run domain tests only if domains are supported Move the domain reload tests to the `tests` directory, merge the `conftest.py` files * Don't run domain reload tests in soft mode, increase timeout --- .github/workflows/main.yml | 2 +- pyproject.toml | 2 +- pythonnet.sln | 2 +- src/domain_tests/conftest.py | 7 ------- tests/conftest.py | 17 +++++++++++++++++ {src => tests}/domain_tests/App.config | 0 .../Python.DomainReloadTests.csproj | 2 +- {src => tests}/domain_tests/TestRunner.cs | 0 .../domain_tests/test_domain_reload.py | 0 9 files changed, 21 insertions(+), 11 deletions(-) delete mode 100644 src/domain_tests/conftest.py rename {src => tests}/domain_tests/App.config (100%) rename {src => tests}/domain_tests/Python.DomainReloadTests.csproj (89%) rename {src => tests}/domain_tests/TestRunner.cs (100%) rename {src => tests}/domain_tests/test_domain_reload.py (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2dd75c529..38782dfb4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: build-test: name: Build and Test runs-on: ${{ matrix.os }}-latest - timeout-minutes: 5 + timeout-minutes: 7 strategy: fail-fast: false diff --git a/pyproject.toml b/pyproject.toml index 9bcf734c6..b6df82f71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,5 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] xfail_strict = true testpaths = [ - "tests", + "tests" ] diff --git a/pythonnet.sln b/pythonnet.sln index e02948c18..eca470595 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -12,7 +12,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "tests\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject diff --git a/src/domain_tests/conftest.py b/src/domain_tests/conftest.py deleted file mode 100644 index 5f0d52e10..000000000 --- a/src/domain_tests/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -from subprocess import check_call -# test_proj_path = os.path.join(cwd, "..", "testing") -cfd = os.path.dirname(__file__) -bin_path = os.path.join(cfd, 'bin') -check_call(["dotnet", "build", cfd, '-o', bin_path]) \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index cf3341f01..0361830d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,8 @@ def pytest_addoption(parser): help="Must be one of default, netcore, netfx and mono" ) +collect_ignore = [] + def pytest_configure(config): global bin_path if "clr" in sys.modules: @@ -73,6 +75,21 @@ def pytest_configure(config): import clr clr.AddReference("Python.Test") + soft_mode = False + try: + os.environ['PYTHONNET_SHUTDOWN_MODE'] == 'Soft' + except: pass + + if config.getoption("--runtime") == "netcore" or soft_mode\ + : + collect_ignore.append("domain_tests/test_domain_reload.py") + else: + domain_tests_dir = os.path.join(os.path.dirname(__file__), "domain_tests") + bin_path = os.path.join(domain_tests_dir, "bin") + check_call(["dotnet", "build", domain_tests_dir, "-o", bin_path]) + + + def pytest_unconfigure(config): global bin_path diff --git a/src/domain_tests/App.config b/tests/domain_tests/App.config similarity index 100% rename from src/domain_tests/App.config rename to tests/domain_tests/App.config diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/tests/domain_tests/Python.DomainReloadTests.csproj similarity index 89% rename from src/domain_tests/Python.DomainReloadTests.csproj rename to tests/domain_tests/Python.DomainReloadTests.csproj index 54196f210..9cb61c6f4 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/tests/domain_tests/Python.DomainReloadTests.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs similarity index 100% rename from src/domain_tests/TestRunner.cs rename to tests/domain_tests/TestRunner.cs diff --git a/src/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py similarity index 100% rename from src/domain_tests/test_domain_reload.py rename to tests/domain_tests/test_domain_reload.py From 5ded48dd3799a7d0982845d107720a5dd505dabf Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 14 May 2021 04:59:11 -0700 Subject: [PATCH 109/151] dispose all temporary objects in PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test --- src/embed_tests/TestRuntime.cs | 44 ++++++++++++++++++---------------- src/runtime/runtime.cs | 15 +++++++++--- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 32369190c..4e05850c1 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -92,27 +92,29 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() { Runtime.Runtime.Py_Initialize(); - // Create an instance of threading.Lock, which is one of the very few types that does not have the - // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. - var threading = Runtime.Runtime.PyImport_ImportModule("threading"); - Exceptions.ErrorCheck(threading); - var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); - Exceptions.ErrorCheck(threadingDict); - var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); - if (lockType.IsNull) - throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); - - var args = Runtime.Runtime.PyTuple_New(0); - var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args); - Runtime.Runtime.XDecref(args); - Exceptions.ErrorCheck(lockInstance); - - Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); - Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); - - threading.Dispose(); - - Runtime.Runtime.Py_Finalize(); + try + { + // Create an instance of threading.Lock, which is one of the very few types that does not have the + // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. + using var threading = Runtime.Runtime.PyImport_ImportModule("threading"); + Exceptions.ErrorCheck(threading); + var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + Exceptions.ErrorCheck(threadingDict); + var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); + if (lockType.IsNull) + throw new PythonException(); + + using var args = NewReference.DangerousFromPointer(Runtime.Runtime.PyTuple_New(0)); + using var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, args); + Exceptions.ErrorCheck(lockInstance); + + Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); + Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); + } + finally + { + Runtime.Runtime.Py_Finalize(); + } } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 6ebf885a0..232dd8708 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1009,6 +1009,11 @@ internal static string PyObject_GetTypeName(IntPtr op) return Marshal.PtrToStringAnsi(ppName); } + /// + /// Test whether the Python object is an iterable. + /// + internal static bool PyObject_IsIterable(BorrowedReference ob) + => PyObject_IsIterable(ob.DangerousGetAddress()); /// /// Test whether the Python object is an iterable. /// @@ -1078,7 +1083,10 @@ internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); - internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) => Delegates.PyObject_CallObject(pointer, args); + internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) + => Delegates.PyObject_CallObject(new BorrowedReference(pointer), new BorrowedReference(args)) + .DangerousMoveToPointerOrNull(); internal static int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); @@ -1880,6 +1888,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) //==================================================================== // Python iterator API //==================================================================== + internal static bool PyIter_Check(BorrowedReference ob) => PyIter_Check(ob.DangerousGetAddress()); internal static bool PyIter_Check(IntPtr pointer) { @@ -2317,7 +2326,7 @@ static Delegates() PyObject_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_DelItem), GetUnmanagedDll(_PythonDll)); PyObject_GetIter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetIter), GetUnmanagedDll(_PythonDll)); PyObject_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Call), GetUnmanagedDll(_PythonDll)); - PyObject_CallObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_CallObject), GetUnmanagedDll(_PythonDll)); + PyObject_CallObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_CallObject), GetUnmanagedDll(_PythonDll)); PyObject_RichCompareBool = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_RichCompareBool), GetUnmanagedDll(_PythonDll)); PyObject_IsInstance = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsInstance), GetUnmanagedDll(_PythonDll)); PyObject_IsSubclass = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsSubclass), GetUnmanagedDll(_PythonDll)); @@ -2616,7 +2625,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_DelItem { get; } internal static delegate* unmanaged[Cdecl] PyObject_GetIter { get; } internal static delegate* unmanaged[Cdecl] PyObject_Call { get; } - internal static delegate* unmanaged[Cdecl] PyObject_CallObject { get; } + internal static delegate* unmanaged[Cdecl] PyObject_CallObject { get; } internal static delegate* unmanaged[Cdecl] PyObject_RichCompareBool { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsInstance { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsSubclass { get; } From 21169db26e88212efc0bebc7578bc92572c7ba31 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 1 Apr 2021 23:17:17 -0700 Subject: [PATCH 110/151] use PyType instances instead of raw pointers in TypeManager type cache and ConstructorBinding instances --- src/runtime/classbase.cs | 4 +- src/runtime/classderived.cs | 6 +- src/runtime/classmanager.cs | 14 +-- src/runtime/clrobject.cs | 2 + src/runtime/constructorbinding.cs | 20 ++-- src/runtime/exceptions.cs | 9 ++ src/runtime/extensiontype.cs | 10 +- src/runtime/managedtype.cs | 35 +++---- src/runtime/metatype.cs | 4 +- src/runtime/pytype.cs | 13 +++ src/runtime/runtime.cs | 22 ++--- src/runtime/runtime_data.cs | 3 +- src/runtime/typemanager.cs | 157 ++++++++++++------------------ 13 files changed, 142 insertions(+), 157 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index bf6a8034d..55f5c5b8f 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -373,7 +373,7 @@ public static int tp_clear(IntPtr ob) protected override void OnSave(InterDomainContext context) { base.OnSave(context); - if (pyHandle != tpHandle) + if (!this.IsClrMetaTypeInstance()) { IntPtr dict = GetObjectDict(pyHandle); Runtime.XIncref(dict); @@ -384,7 +384,7 @@ protected override void OnSave(InterDomainContext context) protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); - if (pyHandle != tpHandle) + if (!this.IsClrMetaTypeInstance()) { IntPtr dict = context.Storage.GetValue("dict"); SetObjectDict(pyHandle, dict); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index f8b97b397..cc2397225 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -122,11 +122,13 @@ internal static IntPtr ToPython(IPythonDerivedType obj) /// internal static Type CreateDerivedType(string name, Type baseType, - IntPtr py_dict, + BorrowedReference dictRef, string namespaceStr, string assemblyName, string moduleName = "Python.Runtime.Dynamic.dll") { + // TODO: clean up + IntPtr py_dict = dictRef.DangerousGetAddress(); if (null != namespaceStr) { name = namespaceStr + "." + name; @@ -824,7 +826,7 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec try { // create the python object - IntPtr type = TypeManager.GetTypeHandle(obj.GetType()); + BorrowedReference type = TypeManager.GetTypeReference(obj.GetType()); self = new CLRObject(obj, type); // set __pyobj__ to self and deref the python object which will allow this diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 306962f56..e3c5b2a3b 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -117,7 +117,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // Python object's dictionary tool; thus raising an AttributeError // instead of a TypeError. // Classes are re-initialized on in RestoreRuntimeData. - var dict = new BorrowedReference(Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict)); + using var dict = Runtime.PyObject_GenericGetDict(cls.Value.TypeReference); foreach (var member in cls.Value.dotNetMembers) { // No need to decref the member, the ClassBase instance does @@ -135,7 +135,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) } } // We modified the Type object, notify it we did. - Runtime.PyType_Modified(cls.Value.tpHandle); + Runtime.PyType_Modified(cls.Value.TypeReference); } } @@ -155,7 +155,7 @@ internal static Dictionary RestoreRuntimeData(R // re-init the class InitClassBase(pair.Key.Value, pair.Value); // We modified the Type object, notify it we did. - Runtime.PyType_Modified(pair.Value.tpHandle); + Runtime.PyType_Modified(pair.Value.TypeReference); var context = contexts[pair.Value.pyHandle]; pair.Value.Load(context); loadedObjs.Add(pair.Value, context); @@ -266,10 +266,10 @@ private static void InitClassBase(Type type, ClassBase impl) // point to the managed methods providing the implementation. - IntPtr tp = TypeManager.GetTypeHandle(impl, type); + var pyType = TypeManager.GetType(impl, type); // Finally, initialize the class __dict__ and return the object. - var dict = new BorrowedReference(Marshal.ReadIntPtr(tp, TypeOffset.tp_dict)); + using var dict = Runtime.PyObject_GenericGetDict(pyType.Reference); if (impl.dotNetMembers == null) @@ -312,7 +312,7 @@ private static void InitClassBase(Type type, ClassBase impl) // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) { - var ctors = new ConstructorBinding(type, tp, co.binder); + var ctors = new ConstructorBinding(type, pyType, co.binder); // ExtensionType types are untracked, so don't Incref() them. // TODO: deprecate __overloads__ soon... Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.ObjectReference); @@ -332,7 +332,7 @@ private static void InitClassBase(Type type, ClassBase impl) // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(tp); + Runtime.PyType_Modified(pyType.Reference); } internal static bool ShouldBindMethod(MethodBase mb) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 46cd896e2..142fade25 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -26,6 +26,8 @@ internal CLRObject(object ob, IntPtr tp) if (ob is Exception e) Exceptions.SetArgsAndCause(e, py); } + internal CLRObject(object ob, BorrowedReference tp) : this(ob, tp.DangerousGetAddress()) { } + protected CLRObject() { } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 803823e39..6706c2b48 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -23,16 +23,16 @@ namespace Python.Runtime internal class ConstructorBinding : ExtensionType { private MaybeType type; // The managed Type being wrapped in a ClassObject - private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. + private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; [NonSerialized] private IntPtr repr; - public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) + public ConstructorBinding(Type type, PyType typeToCreate, ConstructorBinder ctorBinder) { this.type = type; - this.pyTypeHndl = pyTypeHndl; // steal a type reference + this.typeToCreate = typeToCreate; this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -110,7 +110,7 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(tp, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.typeToCreate, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -169,7 +169,7 @@ public static int tp_clear(IntPtr ob) public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ConstructorBinding)GetManagedObject(ob); - int res = PyVisit(self.pyTypeHndl, visit, arg); + int res = PyVisit(self.typeToCreate.Handle, visit, arg); if (res != 0) return res; res = PyVisit(self.repr, visit, arg); @@ -190,15 +190,15 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject - private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. + private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; private ConstructorInfo ctorInfo; private IntPtr repr; - public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) + public BoundContructor(Type type, PyType typeToCreate, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - this.pyTypeHndl = pyTypeHndl; // steal a type reference + this.typeToCreate = typeToCreate; this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -229,7 +229,7 @@ public static IntPtr tp_call(IntPtr op, IntPtr args, IntPtr kw) } // Instantiate the python object that wraps the result of the method call // and return the PyObject* to it. - return CLRObject.GetInstHandle(obj, self.pyTypeHndl); + return CLRObject.GetInstHandle(obj, self.typeToCreate.Reference).DangerousMoveToPointer(); } /// @@ -272,7 +272,7 @@ public static int tp_clear(IntPtr ob) public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (BoundContructor)GetManagedObject(ob); - int res = PyVisit(self.pyTypeHndl, visit, arg); + int res = PyVisit(self.typeToCreate.Handle, visit, arg); if (res != 0) return res; res = PyVisit(self.repr, visit, arg); diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 40e018fe6..163b0a11e 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -206,6 +206,15 @@ internal static void ErrorOccurredCheck(IntPtr pointer) } } + internal static IntPtr ErrorCheckIfNull(IntPtr pointer) + { + if (pointer == IntPtr.Zero && ErrorOccurred()) + { + throw new PythonException(); + } + return pointer; + } + /// /// ExceptionMatches Method /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 554837c46..34a82fe37 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -18,7 +18,7 @@ public ExtensionType() // The Python instance object is related to an instance of a // particular concrete subclass with a hidden CLR gchandle. - IntPtr tp = TypeManager.GetTypeHandle(GetType()); + BorrowedReference tp = TypeManager.GetTypeReference(GetType()); //int rc = (int)Marshal.ReadIntPtr(tp, TypeOffset.ob_refcnt); //if (rc > 1050) @@ -27,11 +27,11 @@ public ExtensionType() // DebugUtil.DumpType(tp); //} - IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); + NewReference py = Runtime.PyType_GenericAlloc(tp, 0); - // Steals a ref to tpHandle. - tpHandle = tp; - pyHandle = py; + // Borrowed reference. Valid as long as pyHandle is valid. + tpHandle = tp.DangerousGetAddress(); + pyHandle = py.DangerousMoveToPointer(); #if DEBUG GetGCHandle(ObjectReference, TypeReference, out var existing); diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 41408abc7..e2f7a171f 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics; @@ -27,8 +28,8 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * - internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); - internal BorrowedReference TypeReference => new BorrowedReference(tpHandle); + internal BorrowedReference ObjectReference => new(pyHandle); + internal BorrowedReference TypeReference => new(tpHandle); private static readonly Dictionary _managedObjs = new Dictionary(); @@ -78,12 +79,12 @@ internal void FreeGCHandle() } } - internal static ManagedType GetManagedObject(BorrowedReference ob) + internal static ManagedType? GetManagedObject(BorrowedReference ob) => GetManagedObject(ob.DangerousGetAddress()); /// /// Given a Python object, return the associated managed object or null. /// - internal static ManagedType GetManagedObject(IntPtr ob) + internal static ManagedType? GetManagedObject(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -106,7 +107,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) /// /// Given a Python object, return the associated managed object type or null. /// - internal static ManagedType GetManagedObjectType(IntPtr ob) + internal static ManagedType? GetManagedObjectType(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -121,18 +122,6 @@ internal static ManagedType GetManagedObjectType(IntPtr ob) return null; } - - internal static ManagedType GetManagedObjectErr(IntPtr ob) - { - ManagedType result = GetManagedObject(ob); - if (result == null) - { - Exceptions.SetError(Exceptions.TypeError, "invalid argument, expected CLR type"); - } - return result; - } - - internal static bool IsInstanceOfManagedType(BorrowedReference ob) => IsInstanceOfManagedType(ob.DangerousGetAddressOrNull()); internal static bool IsInstanceOfManagedType(IntPtr ob) @@ -156,6 +145,13 @@ internal static bool IsManagedType(BorrowedReference type) return (flags & TypeFlags.HasClrInstance) != 0; } + public bool IsClrMetaTypeInstance() + { + Debug.Assert(Runtime.PyCLRMetaType != IntPtr.Zero); + Debug.Assert(pyHandle != IntPtr.Zero); + return Runtime.PyObject_TYPE(pyHandle) == Runtime.PyCLRMetaType; + } + internal static IDictionary GetManagedObjects() { return _managedObjs; @@ -185,7 +181,8 @@ internal void CallTypeClear() { return; } - var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + + var clearPtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_clear); if (clearPtr == IntPtr.Zero) { return; @@ -203,7 +200,7 @@ internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) { return; } - var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + var traversePtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_traverse); if (traversePtr == IntPtr.Zero) { return; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 1fde7dd78..a25df30e5 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -132,7 +132,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) { - return TypeManager.CreateSubType(name, base_type, dict); + return TypeManager.CreateSubType(name, base_type, clsDict.Reference); } } } @@ -266,7 +266,7 @@ public static int tp_setattro(IntPtr tp, IntPtr name, IntPtr value) } int res = Runtime.PyObject_GenericSetAttr(tp, name, value); - Runtime.PyType_Modified(tp); + Runtime.PyType_Modified(new BorrowedReference(tp)); return res; } diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs index 8bc08b76d..e3d95db8d 100644 --- a/src/runtime/pytype.cs +++ b/src/runtime/pytype.cs @@ -6,6 +6,7 @@ namespace Python.Runtime { + [Serializable] public class PyType : PyObject { /// Creates heap type object from the . @@ -13,6 +14,12 @@ public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases) /// Wraps an existing type object. public PyType(PyObject o) : base(FromObject(o)) { } + internal PyType(BorrowedReference reference) : base(reference) + { + if (!Runtime.PyType_Check(this.Handle)) + throw new ArgumentException("object is not a type"); + } + /// Checks if specified object is a Python type. public static bool IsType(PyObject value) { @@ -21,6 +28,12 @@ public static bool IsType(PyObject value) return Runtime.PyType_Check(value.obj); } + internal IntPtr GetSlot(TypeSlotID slot) + { + IntPtr result = Runtime.PyType_GetSlot(this.Reference, slot); + return Exceptions.ErrorCheckIfNull(result); + } + private static BorrowedReference FromObject(PyObject o) { if (o is null) throw new ArgumentNullException(nameof(o)); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 232dd8708..8a3ad9231 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1991,7 +1991,7 @@ internal static bool PyType_Check(IntPtr ob) } - internal static void PyType_Modified(IntPtr type) => Delegates.PyType_Modified(type); + internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); internal static bool PyType_IsSubtype(BorrowedReference t1, IntPtr ofType) => PyType_IsSubtype(t1, new BorrowedReference(ofType)); internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) => Delegates.PyType_IsSubtype(t1, t2); @@ -2014,14 +2014,10 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe internal static IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw) => Delegates.PyType_GenericNew(type, args, kw); - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) - { - return PyType_GenericAlloc(type, new IntPtr(n)); - } - + internal static IntPtr PyType_GenericAlloc(IntPtr type, nint n) => PyType_GenericAlloc(new BorrowedReference(type), n).DangerousMoveToPointer(); + internal static NewReference PyType_GenericAlloc(BorrowedReference type, nint n) => Delegates.PyType_GenericAlloc(type, n); - private static IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n) => Delegates.PyType_GenericAlloc(type, n); - + internal static IntPtr PyType_GetSlot(BorrowedReference type, TypeSlotID slot) => Delegates.PyType_GetSlot(type, slot); internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); /// @@ -2489,10 +2485,10 @@ static Delegates() PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); PySys_GetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_GetObject), GetUnmanagedDll(_PythonDll)); PySys_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetObject), GetUnmanagedDll(_PythonDll)); - PyType_Modified = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Modified), GetUnmanagedDll(_PythonDll)); + PyType_Modified = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Modified), GetUnmanagedDll(_PythonDll)); PyType_IsSubtype = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_IsSubtype), GetUnmanagedDll(_PythonDll)); PyType_GenericNew = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericNew), GetUnmanagedDll(_PythonDll)); - PyType_GenericAlloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericAlloc), GetUnmanagedDll(_PythonDll)); + PyType_GenericAlloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericAlloc), GetUnmanagedDll(_PythonDll)); PyType_Ready = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Ready), GetUnmanagedDll(_PythonDll)); _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); @@ -2534,6 +2530,7 @@ static Delegates() PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyType_GetSlot = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GetSlot), GetUnmanagedDll(_PythonDll)); PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); try @@ -2774,10 +2771,10 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } internal static delegate* unmanaged[Cdecl] PySys_GetObject { get; } internal static delegate* unmanaged[Cdecl] PySys_SetObject { get; } - internal static delegate* unmanaged[Cdecl] PyType_Modified { get; } + internal static delegate* unmanaged[Cdecl] PyType_Modified { get; } internal static delegate* unmanaged[Cdecl] PyType_IsSubtype { get; } internal static delegate* unmanaged[Cdecl] PyType_GenericNew { get; } - internal static delegate* unmanaged[Cdecl] PyType_GenericAlloc { get; } + internal static delegate* unmanaged[Cdecl] PyType_GenericAlloc { get; } internal static delegate* unmanaged[Cdecl] PyType_Ready { get; } internal static delegate* unmanaged[Cdecl] _PyType_Lookup { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } @@ -2819,6 +2816,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } + internal static delegate* unmanaged[Cdecl] PyType_GetSlot { get; } internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; } } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 29cea4181..832e5bbec 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -119,12 +119,13 @@ private static void RestoreRuntimeDataImpl() var formatter = CreateFormatter(); var storage = (RuntimeDataStorage)formatter.Deserialize(ms); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); RestoreRuntimeDataModules(storage.GetStorage("modules")); TypeManager.RestoreRuntimeData(storage.GetStorage("types")); var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); ImportHook.RestoreRuntimeData(storage.GetStorage("import")); - PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); foreach (var item in objs) { diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 20b95d85c..9707568b5 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -21,7 +21,7 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache = new Dictionary(); + private static Dictionary cache = new(); private static readonly Dictionary _slotsHolders = new Dictionary(); private static Dictionary _slotsImpls = new Dictionary(); @@ -45,19 +45,19 @@ internal static void Initialize() internal static void RemoveTypes() { - foreach (var tpHandle in cache.Values) + foreach (var type in cache.Values) { SlotsHolder holder; - if (_slotsHolders.TryGetValue(tpHandle, out holder)) + if (_slotsHolders.TryGetValue(type.Handle, out holder)) { // If refcount > 1, it needs to reset the managed slot, // otherwise it can dealloc without any trick. - if (Runtime.Refcount(tpHandle) > 1) + if (Runtime.Refcount(type.Handle) > 1) { holder.ResetSlots(); } } - Runtime.XDecref(tpHandle); + type.Dispose(); } cache.Clear(); _slotsImpls.Clear(); @@ -68,7 +68,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) { foreach (var tpHandle in cache.Values) { - Runtime.XIncref(tpHandle); + Runtime.XIncref(tpHandle.Handle); } storage.AddValue("cache", cache); storage.AddValue("slots", _slotsImpls); @@ -78,45 +78,33 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - storage.GetValue>("cache", out var _cache); + storage.GetValue>("cache", out var _cache); foreach (var entry in _cache) { if (!entry.Key.Valid) { - Runtime.XDecref(entry.Value); + entry.Value.Dispose(); continue; } Type type = entry.Key.Value;; - IntPtr handle = entry.Value; - cache[type] = handle; - SlotsHolder holder = CreateSolotsHolder(handle); - InitializeSlots(handle, _slotsImpls[type], holder); + cache[type] = entry.Value; + SlotsHolder holder = CreateSolotsHolder(entry.Value.Handle); + InitializeSlots(entry.Value.Handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) } } - /// - /// Return value: Borrowed reference. - /// Given a managed Type derived from ExtensionType, get the handle to - /// a Python type object that delegates its implementation to the Type - /// object. These Python type instances are used to implement internal - /// descriptor and utility types like ModuleObject, PropertyObject, etc. - /// - [Obsolete] - internal static IntPtr GetTypeHandle(Type type) + internal static PyType GetType(Type type) { // Note that these types are cached with a refcount of 1, so they // effectively exist until the CPython runtime is finalized. - IntPtr handle; - cache.TryGetValue(type, out handle); - if (handle != IntPtr.Zero) + if (!cache.TryGetValue(type, out var pyType)) { - return handle; + pyType = CreateType(type); + cache[type] = pyType; + _slotsImpls.Add(type, type); } - handle = CreateType(type); - cache[type] = handle; - _slotsImpls.Add(type, type); - return handle; + return pyType; } /// /// Given a managed Type derived from ExtensionType, get the handle to @@ -124,28 +112,23 @@ internal static IntPtr GetTypeHandle(Type type) /// object. These Python type instances are used to implement internal /// descriptor and utility types like ModuleObject, PropertyObject, etc. /// - internal static BorrowedReference GetTypeReference(Type type) - => new BorrowedReference(GetTypeHandle(type)); + internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; /// - /// Return value: Borrowed reference. - /// Get the handle of a Python type that reflects the given CLR type. + /// Get the Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. /// - internal static IntPtr GetTypeHandle(ClassBase obj, Type type) + internal static PyType GetType(ClassBase obj, Type type) { - IntPtr handle; - cache.TryGetValue(type, out handle); - if (handle != IntPtr.Zero) + if (!cache.TryGetValue(type, out var pyType)) { - return handle; + pyType = CreateType(obj, type); + cache[type] = pyType; + _slotsImpls.Add(type, obj.GetType()); } - handle = CreateType(obj, type); - cache[type] = handle; - _slotsImpls.Add(type, obj.GetType()); - return handle; + return pyType; } @@ -157,7 +140,7 @@ internal static IntPtr GetTypeHandle(ClassBase obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// - internal static unsafe IntPtr CreateType(Type impl) + internal static unsafe PyType CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); IntPtr base_ = impl == typeof(CLRModule) @@ -187,21 +170,27 @@ internal static unsafe IntPtr CreateType(Type impl) throw new PythonException(); } - var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); + // TODO: use PyType(TypeSpec) constructor + var pyType = new PyType(new BorrowedReference(type)); + Runtime.XDecref(type); + + NewReference dict = Runtime.PyObject_GenericGetDict(pyType.Reference); var mod = NewReference.DangerousFromPointer(Runtime.PyString_FromString("CLR")); Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); mod.Dispose(); - InitMethods(type, impl); + InitMethods(dict, impl); + + dict.Dispose(); // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(type); - return type; + Runtime.PyType_Modified(pyType.Reference); + return pyType; } - internal static IntPtr CreateType(ClassBase impl, Type clrType) + internal static PyType CreateType(ClassBase impl, Type clrType) { // Cleanup the type name to get rid of funny nested type names. string name = $"clr.{clrType.FullName}"; @@ -322,8 +311,9 @@ internal static IntPtr CreateType(ClassBase impl, Type clrType) impl.pyHandle = type; //DebugUtil.DumpType(type); - - return type; + var pyType = new PyType(new BorrowedReference(type)); + Runtime.XDecref(type); + return pyType; } static int InheritOrAllocateStandardFields(IntPtr type, IntPtr @base) @@ -351,9 +341,8 @@ void InheritOrAllocate(int typeField) return newFieldOffset; } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, BorrowedReference dictRef) { - var dictRef = new BorrowedReference(py_dict); // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation string name = Runtime.GetManagedString(py_name); @@ -400,18 +389,18 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type.Value, - py_dict, + dictRef, (string)namespaceStr, (string)assembly); // create the new ManagedType and python type ClassBase subClass = ClassManager.GetClass(subType); - IntPtr py_type = GetTypeHandle(subClass, subType); + IntPtr py_type = GetType(subClass, subType).Handle; // by default the class dict will have all the C# methods in it, but as this is a // derived class we want the python overrides in there instead if they exist. var cls_dict = new BorrowedReference(Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict)); - ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, new BorrowedReference(py_dict))); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, dictRef)); Runtime.XIncref(py_type); // Update the __classcell__ if it exists BorrowedReference cell = Runtime.PyDict_GetItemString(cls_dict, "__classcell__"); @@ -513,7 +502,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(type); + Runtime.PyType_Modified(new BorrowedReference(type)); //DebugUtil.DumpType(type); return type; @@ -576,7 +565,6 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, return mdef; } - /// /// Utility method to allocate a type object & do basic initialization. /// @@ -599,6 +587,8 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); + + #warning dead code? temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); @@ -610,6 +600,7 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) temp = type + TypeOffset.bf_getbuffer; Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + return type; } @@ -661,45 +652,24 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo } } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - /// Can override the slot when it existed - static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder) { - var offset = TypeOffset.GetSlotOffset(name); - if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + if (!Enum.TryParse(name, out var id)) { - return; + throw new NotSupportedException("Bad slot name " + name); } - Marshal.WriteIntPtr(type, offset, slot); + int offset = TypeOffset.GetSlotOffset(name); + InitializeSlot(type, offset, thunk, slotsHolder); } - static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder) { - int offset = TypeOffset.GetSlotOffset(name); - - if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) - { - return; - } - Marshal.WriteIntPtr(type, offset, thunk.Address); - if (slotsHolder != null) - { - slotsHolder.Set(offset, thunk); - } + var thunk = Interop.GetThunk(method); + InitializeSlot(type, slotOffset, thunk, slotsHolder); } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + static void InitializeSlot(IntPtr type, int slotOffset, ThunkInfo thunk, SlotsHolder slotsHolder) { - var thunk = Interop.GetThunk(method); Marshal.WriteIntPtr(type, slotOffset, thunk.Address); if (slotsHolder != null) { @@ -707,20 +677,13 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots } } - static bool IsSlotSet(IntPtr type, string name) - { - int offset = TypeOffset.GetSlotOffset(name); - return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; - } - /// - /// Given a newly allocated Python type object and a managed Type that + /// Given a dict of a newly allocated Python type object and a managed Type that /// implements it, initialize any methods defined by the Type that need /// to appear in the Python type __dict__ (based on custom attribute). /// - private static void InitMethods(IntPtr pytype, Type type) + private static void InitMethods(BorrowedReference typeDict, Type type) { - IntPtr dict = Marshal.ReadIntPtr(pytype, TypeOffset.tp_dict); Type marker = typeof(PythonMethodAttribute); BindingFlags flags = BindingFlags.Public | BindingFlags.Static; @@ -740,7 +703,7 @@ private static void InitMethods(IntPtr pytype, Type type) var mi = new MethodInfo[1]; mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); - Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + Runtime.PyDict_SetItemString(typeDict, method_name, m.ObjectReference); m.DecrRefCount(); addedMethods.Add(method_name); } From 2c6202863d2d960f3d9746f00fcd8f1ce564deae Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 18:09:39 -0700 Subject: [PATCH 111/151] + class diagram for ManagedType and subclasses --- src/runtime/ManagedTypes.cd | 165 ++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/runtime/ManagedTypes.cd diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd new file mode 100644 index 000000000..9c0f73b4f --- /dev/null +++ b/src/runtime/ManagedTypes.cd @@ -0,0 +1,165 @@ + + + + + + FAAAAgAIAAAEDAAAAAAAAEACIACJAAIAAAAAAAIAAAQ= + classbase.cs + + + + + + AAAAAABIAAABDAAAIAIAAAAAAAAAAAAAAAAACAiAAAQ= + classderived.cs + + + + + + AAAAAAAAABAAAAAAAAAAACAAIAAJAAAAIAAAAACAAAI= + arrayobject.cs + + + + + + AAABAAAIAAAAAAAAIAAAAAAAAAAAAIAAACAAAACAAAA= + classobject.cs + + + + + + AAAACAAAAAAABABAAAAACAAAABAJAAAAAAAAAAIAAAQ= + constructorbinding.cs + + + + + + EAAAAAAAAAAAAAAAAAACAAACBIAAAAJAAAAAAAAAAAA= + clrobject.cs + + + + + + AAAAAEAgIAQABAAAAABAAAAAIAIAAAAAAhAQAAAAKBA= + moduleobject.cs + + + + + + AAAACAAAAAAABAAAAAAACAAAABAJAAAAAAAAAAIAEAQ= + constructorbinding.cs + + + + + + AAABAAAAAAAAAABAAAAAAEAAIAACAAAAAAAAAACAAAA= + delegateobject.cs + + + + + + AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= + eventbinding.cs + + + + + + AAACAAAAAAAAAAAAAAIAAIAAAAAEAAAAQABAAAIBEAQ= + eventobject.cs + + + + + + AAAAAgAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + exceptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAECAAAAAEEBAAAAAAABAAQ= + extensiontype.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAIBEAA= + fieldobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAggAAAAAAAEAACAACAAAA= + interfaceobject.cs + + + + + + UCBBgoBAIUgAAAEAACAAsAACAgAIABIAQYAAACIYIBA= + managedtype.cs + + + + + + AQAAAAAICBAAAQBAAAABAAIAAgABAAABAAAAUBCAAAQ= + metatype.cs + + + + + + EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= + methodbinding.cs + + + + + + FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= + methodobject.cs + + + + + + AAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + modulefunctionobject.cs + + + + + + ECCCCkAAAAAABAAAAAABAAACAAAIAIIAEAAAAAIACAQ= + moduleobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + modulepropertyobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAQAAAAAIBEAg= + propertyobject.cs + + + + \ No newline at end of file From 91e98a2d9683dc25257dcd93b057651b5ad1f0e4 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 20:08:25 -0700 Subject: [PATCH 112/151] added OverloadMapper to ManagedTypes class diagram --- src/runtime/ManagedTypes.cd | 53 +++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd index 9c0f73b4f..385ae7117 100644 --- a/src/runtime/ManagedTypes.cd +++ b/src/runtime/ManagedTypes.cd @@ -29,9 +29,9 @@ - + - AAAACAAAAAAABABAAAAACAAAABAJAAAAAAAAAAIAAAQ= + AAAACAAAAAAABABAAAAACAAAABAJAEAAAAAAAAIAAAA= constructorbinding.cs @@ -52,7 +52,7 @@ - AAAACAAAAAAABAAAAAAACAAAABAJAAAAAAAAAAIAEAQ= + AAAACAAAAAAABAAAAAAACAAAABAJAEAAAAAAAAIAEAA= constructorbinding.cs @@ -64,7 +64,15 @@ - + + + + + + + + + AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= eventbinding.cs @@ -87,12 +95,12 @@ - AAAAAAAAAAAAAAAAAAAAAAECAAAAAEEBAAAAAAABAAQ= + AAAAAAAAAAAAAAAAAAAAAAACAAAAAEEBAAAAAAABAAQ= extensiontype.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAIBEAA= fieldobject.cs @@ -120,21 +128,29 @@ - + + + + + + + + + EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= methodbinding.cs - + FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= methodobject.cs - + AAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAIAAAA= modulefunctionobject.cs @@ -148,18 +164,33 @@ - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= modulepropertyobject.cs - + AAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAQAAAAAIBEAg= propertyobject.cs + + + + + + + + + + + + AAAAAAAAAAAAAAAAIAAAAAAAAAABAAAAAgAAAAIAAAQ= + overload.cs + + \ No newline at end of file From 88b19cf1ad8dd0ed48bda1c6dea3370e6ea78c30 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 20:10:19 -0700 Subject: [PATCH 113/151] refactored tp_dealloc in ExtensionType and descendants --- src/runtime/constructorbinding.cs | 20 ++++++------------- src/runtime/eventbinding.cs | 11 +++-------- src/runtime/eventobject.cs | 13 +++++-------- src/runtime/extensiontype.cs | 18 +++++------------ src/runtime/methodbinding.cs | 24 ++++++++++------------- src/runtime/methodobject.cs | 32 ++++++++++++++----------------- src/runtime/moduleobject.cs | 13 ++++++------- src/runtime/native/TypeOffset.cs | 1 - src/runtime/overload.cs | 10 +++------- 9 files changed, 52 insertions(+), 90 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 6706c2b48..ab4a7af0a 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -149,14 +149,10 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.repr); + base.Dealloc(); } public static int tp_clear(IntPtr ob) @@ -252,14 +248,10 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.repr); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 3f5b7b007..60b9bba92 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -103,15 +103,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString(s); } - - /// - /// EventBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (EventBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.target); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 4dc785ddd..e9bd98821 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -198,17 +198,14 @@ public static IntPtr tp_repr(IntPtr ob) } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (EventObject)GetManagedObject(ob); - if (self.unbound != null) + if (this.unbound is not null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; } - self.Dealloc(); + base.Dealloc(); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 34a82fe37..db9eb0f72 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -54,20 +54,12 @@ void SetupGc () } - /// - /// Common finalization code to support custom tp_deallocs. - /// - public static void FinalizeObject(ManagedType self) + protected virtual void Dealloc() { - ClearObjectDict(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); + ClearObjectDict(this.pyHandle); + Runtime.PyObject_GC_Del(this.pyHandle); // Not necessary for decref of `tpHandle`. - self.FreeGCHandle(); - } - - protected void Dealloc() - { - FinalizeObject(this); + this.FreeGCHandle(); } /// @@ -104,7 +96,7 @@ public static void tp_dealloc(IntPtr ob) // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. var self = (ExtensionType)GetManagedObject(ob); - self.Dealloc(); + self?.Dealloc(); } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f33015ba4..783717189 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -31,7 +31,7 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) { Runtime.XIncref(targetType); } - + this.targetType = targetType; this.info = null; @@ -42,12 +42,6 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref target); - Runtime.Py_CLEAR(ref targetType); - } - /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -235,14 +229,16 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - /// - /// MethodBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + private void ClearMembers() { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - self.Dealloc(); + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); + } + + protected override void Dealloc() + { + this.ClearMembers(); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 37c01f5c5..5fa965f1b 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -120,16 +120,6 @@ internal bool IsStatic() return is_static; } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref doc); - if (unbound != null) - { - Runtime.XDecref(unbound.pyHandle); - unbound = null; - } - } - /// /// Descriptor __getattribute__ implementation. /// @@ -210,15 +200,21 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + private void ClearMembers() { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - self.Dealloc(); + Runtime.Py_CLEAR(ref doc); + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + + protected override void Dealloc() + { + this.ClearMembers(); + ClearObjectDict(this.pyHandle); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 3c4e02a23..1ab014c2a 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -274,7 +274,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) Exceptions.SetError(e); return IntPtr.Zero; } - + if (attr == null) { @@ -295,11 +295,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (ModuleObject)GetManagedObject(ob); - tp_clear(ob); - self.Dealloc(); + tp_clear(this.pyHandle); + base.Dealloc(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) @@ -345,7 +344,7 @@ protected override void OnSave(InterDomainContext context) if ((Runtime.PyDict_DelItemString(DictRef, pair.Key) == -1) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { - // Trying to remove a key that's not in the dictionary + // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. Runtime.PyErr_Clear(); } @@ -496,7 +495,7 @@ public static Assembly AddReference(string name) /// clr.GetClrType(IComparable) gives you the Type for IComparable, /// that you can e.g. perform reflection on. Similar to typeof(IComparable) in C# /// or clr.GetClrType(IComparable) in IronPython. - /// + /// /// /// /// The Type object diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 4e5a726bc..a73a9ae43 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -153,7 +153,6 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "__instancecheck__", "__subclasscheck__", "AddReference", - "FinalizeObject", "FindAssembly", "get_SuppressDocs", "get_SuppressOverloads", diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index e9fa91d3b..48fabca4a 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -58,14 +58,10 @@ public static IntPtr tp_repr(IntPtr op) return doc; } - /// - /// OverloadMapper dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (OverloadMapper)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.target); + base.Dealloc(); } } } From 786b450fd40077d04f5c00f8cc7437107f768225 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 22:39:57 -0700 Subject: [PATCH 114/151] refactored tp_clear in ExtensionType and descendants into a virtual C# function Clear --- src/runtime/constructorbinding.cs | 14 ++++++-------- src/runtime/eventbinding.cs | 7 +++---- src/runtime/eventobject.cs | 10 ++++++++++ src/runtime/extensiontype.cs | 13 ++++++++++--- src/runtime/methodbinding.cs | 7 +++---- src/runtime/methodobject.cs | 9 ++++----- src/runtime/moduleobject.cs | 25 ++++++++++++------------- src/runtime/overload.cs | 6 ++++++ 8 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index ab4a7af0a..da35aafd1 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -155,11 +155,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) @@ -254,11 +253,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 60b9bba92..8fb0ce250 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -109,11 +109,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (EventBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.target); - return 0; + Runtime.Py_CLEAR(ref this.target); + base.Clear(); } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index e9bd98821..a0f06c375 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -207,6 +207,16 @@ protected override void Dealloc() } base.Dealloc(); } + + protected override void Clear() + { + if (this.unbound is not null) + { + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; + } + base.Clear(); + } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index db9eb0f72..38fe238de 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -62,6 +62,9 @@ protected virtual void Dealloc() this.FreeGCHandle(); } + /// DecRefs and nulls any fields pointing back to Python + protected virtual void Clear() { } + /// /// Type __setattr__ implementation. /// @@ -88,9 +91,6 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } - /// - /// Default dealloc implementation. - /// public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This @@ -99,6 +99,13 @@ public static void tp_dealloc(IntPtr ob) self?.Dealloc(); } + public static int tp_clear(IntPtr ob) + { + var self = (ExtensionType)GetManagedObject(ob); + self?.Clear(); + return 0; + } + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 783717189..a842dd308 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -241,11 +241,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - return 0; + this.ClearMembers(); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 5fa965f1b..c181c5df1 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -217,12 +217,11 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - return 0; + this.ClearMembers(); + ClearObjectDict(this.pyHandle); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 1ab014c2a..b2721cc47 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -295,12 +295,6 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - protected override void Dealloc() - { - tp_clear(this.pyHandle); - base.Dealloc(); - } - public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ModuleObject)GetManagedObject(ob); @@ -314,17 +308,22 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - public static int tp_clear(IntPtr ob) + protected override void Dealloc() { - var self = (ModuleObject)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.dict); - ClearObjectDict(ob); - foreach (var attr in self.cache.Values) + tp_clear(this.pyHandle); + base.Dealloc(); + } + + protected override void Clear() + { + Runtime.Py_CLEAR(ref this.dict); + ClearObjectDict(this.pyHandle); + foreach (var attr in this.cache.Values) { Runtime.XDecref(attr.pyHandle); } - self.cache.Clear(); - return 0; + this.cache.Clear(); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 48fabca4a..c6c3158fb 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -63,5 +63,11 @@ protected override void Dealloc() Runtime.Py_CLEAR(ref this.target); base.Dealloc(); } + + protected override void Clear() + { + Runtime.Py_CLEAR(ref this.target); + base.Clear(); + } } } From 993707efba6cb713e6e52377d6acb1ed990bc958 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 23:55:52 -0700 Subject: [PATCH 115/151] all Dealloc overrides simply duplicate Clear, so just call both from tp_dealloc and don't override Dealloc --- src/runtime/constructorbinding.cs | 12 ------------ src/runtime/eventbinding.cs | 6 ------ src/runtime/eventobject.cs | 10 ---------- src/runtime/extensiontype.cs | 15 ++++++++++++--- src/runtime/methodbinding.cs | 15 ++------------- src/runtime/methodobject.cs | 21 +++++---------------- src/runtime/moduleobject.cs | 6 ------ src/runtime/overload.cs | 6 ------ 8 files changed, 19 insertions(+), 72 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index da35aafd1..9ac1adc0f 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -149,12 +149,6 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.repr); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.repr); @@ -247,12 +241,6 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.repr); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.repr); diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 8fb0ce250..65c8fdccf 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -103,12 +103,6 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString(s); } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.target); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.target); diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index a0f06c375..941bbdf46 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -198,16 +198,6 @@ public static IntPtr tp_repr(IntPtr ob) } - protected override void Dealloc() - { - if (this.unbound is not null) - { - Runtime.XDecref(this.unbound.pyHandle); - this.unbound = null; - } - base.Dealloc(); - } - protected override void Clear() { if (this.unbound is not null) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 38fe238de..78df805ee 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -56,14 +56,22 @@ void SetupGc () protected virtual void Dealloc() { - ClearObjectDict(this.pyHandle); + var type = Runtime.PyObject_TYPE(this.ObjectReference); Runtime.PyObject_GC_Del(this.pyHandle); - // Not necessary for decref of `tpHandle`. + // Not necessary for decref of `tpHandle` - it is borrowed + this.FreeGCHandle(); + + // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc + Runtime.XDecref(type.DangerousGetAddress()); } /// DecRefs and nulls any fields pointing back to Python - protected virtual void Clear() { } + protected virtual void Clear() + { + ClearObjectDict(this.pyHandle); + // Not necessary for decref of `tpHandle` - it is borrowed + } /// /// Type __setattr__ implementation. @@ -96,6 +104,7 @@ public static void tp_dealloc(IntPtr ob) // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. var self = (ExtensionType)GetManagedObject(ob); + self?.Clear(); self?.Dealloc(); } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index a842dd308..c1e729f9e 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -229,21 +229,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref target); - Runtime.Py_CLEAR(ref targetType); - } - - protected override void Dealloc() - { - this.ClearMembers(); - base.Dealloc(); - } - protected override void Clear() { - this.ClearMembers(); + Runtime.Py_CLEAR(ref this.target); + Runtime.Py_CLEAR(ref this.targetType); base.Clear(); } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index c181c5df1..2787ec999 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -200,26 +200,15 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - private void ClearMembers() + protected override void Clear() { - Runtime.Py_CLEAR(ref doc); - if (unbound != null) + Runtime.Py_CLEAR(ref this.doc); + if (this.unbound != null) { - Runtime.XDecref(unbound.pyHandle); - unbound = null; + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; } - } - protected override void Dealloc() - { - this.ClearMembers(); - ClearObjectDict(this.pyHandle); - base.Dealloc(); - } - - protected override void Clear() - { - this.ClearMembers(); ClearObjectDict(this.pyHandle); base.Clear(); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index b2721cc47..606fa7be9 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -308,12 +308,6 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - protected override void Dealloc() - { - tp_clear(this.pyHandle); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.dict); diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index c6c3158fb..8222dc136 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -58,12 +58,6 @@ public static IntPtr tp_repr(IntPtr op) return doc; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.target); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.target); From 7eac8868338651d36aaac0fe0406f4ae1a74f155 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 21 May 2021 16:08:05 -0700 Subject: [PATCH 116/151] handle ProcessExit event to avoid Python crash when CLR (in particular Mono) is unloaded before Python stops --- src/runtime/pythonengine.cs | 7 +++++++ src/runtime/runtime.cs | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 5925880c0..bab5db1d8 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -192,6 +192,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // Make sure we clean up properly on app domain unload. AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; // The global scope gets used implicitly quite early on, remember // to clear it out when we shut down. @@ -249,6 +250,11 @@ static void OnDomainUnload(object _, EventArgs __) Shutdown(); } + static void OnProcessExit(object _, EventArgs __) + { + Shutdown(); + } + /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -319,6 +325,7 @@ public static void Shutdown(ShutdownMode mode) // If the shutdown handlers trigger a domain unload, // don't call shutdown again. AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; PyScopeManager.Global.Clear(); ExecuteShutdownHandlers(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 8a3ad9231..69d4460a7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -393,6 +393,10 @@ internal static void Shutdown(ShutdownMode mode) { Py_Finalize(); } + else + { + PyGILState_Release(state); + } } } From 2e57b0419bb49fb95165c1641b36c4e389af037b Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 18:09:39 -0700 Subject: [PATCH 117/151] + class diagram for ManagedType and subclasses --- src/runtime/ManagedTypes.cd | 165 ++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/runtime/ManagedTypes.cd diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd new file mode 100644 index 000000000..9c0f73b4f --- /dev/null +++ b/src/runtime/ManagedTypes.cd @@ -0,0 +1,165 @@ + + + + + + FAAAAgAIAAAEDAAAAAAAAEACIACJAAIAAAAAAAIAAAQ= + classbase.cs + + + + + + AAAAAABIAAABDAAAIAIAAAAAAAAAAAAAAAAACAiAAAQ= + classderived.cs + + + + + + AAAAAAAAABAAAAAAAAAAACAAIAAJAAAAIAAAAACAAAI= + arrayobject.cs + + + + + + AAABAAAIAAAAAAAAIAAAAAAAAAAAAIAAACAAAACAAAA= + classobject.cs + + + + + + AAAACAAAAAAABABAAAAACAAAABAJAAAAAAAAAAIAAAQ= + constructorbinding.cs + + + + + + EAAAAAAAAAAAAAAAAAACAAACBIAAAAJAAAAAAAAAAAA= + clrobject.cs + + + + + + AAAAAEAgIAQABAAAAABAAAAAIAIAAAAAAhAQAAAAKBA= + moduleobject.cs + + + + + + AAAACAAAAAAABAAAAAAACAAAABAJAAAAAAAAAAIAEAQ= + constructorbinding.cs + + + + + + AAABAAAAAAAAAABAAAAAAEAAIAACAAAAAAAAAACAAAA= + delegateobject.cs + + + + + + AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= + eventbinding.cs + + + + + + AAACAAAAAAAAAAAAAAIAAIAAAAAEAAAAQABAAAIBEAQ= + eventobject.cs + + + + + + AAAAAgAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + exceptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAECAAAAAEEBAAAAAAABAAQ= + extensiontype.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAIBEAA= + fieldobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAggAAAAAAAEAACAACAAAA= + interfaceobject.cs + + + + + + UCBBgoBAIUgAAAEAACAAsAACAgAIABIAQYAAACIYIBA= + managedtype.cs + + + + + + AQAAAAAICBAAAQBAAAABAAIAAgABAAABAAAAUBCAAAQ= + metatype.cs + + + + + + EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= + methodbinding.cs + + + + + + FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= + methodobject.cs + + + + + + AAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + modulefunctionobject.cs + + + + + + ECCCCkAAAAAABAAAAAABAAACAAAIAIIAEAAAAAIACAQ= + moduleobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + modulepropertyobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAQAAAAAIBEAg= + propertyobject.cs + + + + \ No newline at end of file From 539ce815de51a935805b7a7baad8757d8004b660 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 20:08:25 -0700 Subject: [PATCH 118/151] added OverloadMapper to ManagedTypes class diagram --- src/runtime/ManagedTypes.cd | 53 +++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd index 9c0f73b4f..385ae7117 100644 --- a/src/runtime/ManagedTypes.cd +++ b/src/runtime/ManagedTypes.cd @@ -29,9 +29,9 @@ - + - AAAACAAAAAAABABAAAAACAAAABAJAAAAAAAAAAIAAAQ= + AAAACAAAAAAABABAAAAACAAAABAJAEAAAAAAAAIAAAA= constructorbinding.cs @@ -52,7 +52,7 @@ - AAAACAAAAAAABAAAAAAACAAAABAJAAAAAAAAAAIAEAQ= + AAAACAAAAAAABAAAAAAACAAAABAJAEAAAAAAAAIAEAA= constructorbinding.cs @@ -64,7 +64,15 @@ - + + + + + + + + + AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= eventbinding.cs @@ -87,12 +95,12 @@ - AAAAAAAAAAAAAAAAAAAAAAECAAAAAEEBAAAAAAABAAQ= + AAAAAAAAAAAAAAAAAAAAAAACAAAAAEEBAAAAAAABAAQ= extensiontype.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAIBEAA= fieldobject.cs @@ -120,21 +128,29 @@ - + + + + + + + + + EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= methodbinding.cs - + FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= methodobject.cs - + AAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAIAAAA= modulefunctionobject.cs @@ -148,18 +164,33 @@ - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= modulepropertyobject.cs - + AAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAQAAAAAIBEAg= propertyobject.cs + + + + + + + + + + + + AAAAAAAAAAAAAAAAIAAAAAAAAAABAAAAAgAAAAIAAAQ= + overload.cs + + \ No newline at end of file From 25e38640c6367c04b644c94b702a5ef2f2e13248 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 20:10:19 -0700 Subject: [PATCH 119/151] refactored tp_dealloc in ExtensionType and descendants --- src/runtime/constructorbinding.cs | 20 ++++++------------- src/runtime/eventbinding.cs | 11 +++-------- src/runtime/eventobject.cs | 13 +++++-------- src/runtime/extensiontype.cs | 18 +++++------------ src/runtime/methodbinding.cs | 24 ++++++++++------------- src/runtime/methodobject.cs | 32 ++++++++++++++----------------- src/runtime/moduleobject.cs | 13 ++++++------- src/runtime/native/TypeOffset.cs | 1 - src/runtime/overload.cs | 10 +++------- 9 files changed, 52 insertions(+), 90 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 6706c2b48..ab4a7af0a 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -149,14 +149,10 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.repr); + base.Dealloc(); } public static int tp_clear(IntPtr ob) @@ -252,14 +248,10 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.repr); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 3f5b7b007..60b9bba92 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -103,15 +103,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString(s); } - - /// - /// EventBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (EventBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.target); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 4dc785ddd..e9bd98821 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -198,17 +198,14 @@ public static IntPtr tp_repr(IntPtr ob) } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (EventObject)GetManagedObject(ob); - if (self.unbound != null) + if (this.unbound is not null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; } - self.Dealloc(); + base.Dealloc(); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 34a82fe37..db9eb0f72 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -54,20 +54,12 @@ void SetupGc () } - /// - /// Common finalization code to support custom tp_deallocs. - /// - public static void FinalizeObject(ManagedType self) + protected virtual void Dealloc() { - ClearObjectDict(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); + ClearObjectDict(this.pyHandle); + Runtime.PyObject_GC_Del(this.pyHandle); // Not necessary for decref of `tpHandle`. - self.FreeGCHandle(); - } - - protected void Dealloc() - { - FinalizeObject(this); + this.FreeGCHandle(); } /// @@ -104,7 +96,7 @@ public static void tp_dealloc(IntPtr ob) // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. var self = (ExtensionType)GetManagedObject(ob); - self.Dealloc(); + self?.Dealloc(); } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f33015ba4..783717189 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -31,7 +31,7 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) { Runtime.XIncref(targetType); } - + this.targetType = targetType; this.info = null; @@ -42,12 +42,6 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref target); - Runtime.Py_CLEAR(ref targetType); - } - /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -235,14 +229,16 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - /// - /// MethodBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + private void ClearMembers() { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - self.Dealloc(); + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); + } + + protected override void Dealloc() + { + this.ClearMembers(); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 37c01f5c5..5fa965f1b 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -120,16 +120,6 @@ internal bool IsStatic() return is_static; } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref doc); - if (unbound != null) - { - Runtime.XDecref(unbound.pyHandle); - unbound = null; - } - } - /// /// Descriptor __getattribute__ implementation. /// @@ -210,15 +200,21 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + private void ClearMembers() { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - self.Dealloc(); + Runtime.Py_CLEAR(ref doc); + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + + protected override void Dealloc() + { + this.ClearMembers(); + ClearObjectDict(this.pyHandle); + base.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8655253df..8892390c3 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -274,7 +274,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) Exceptions.SetError(e); return IntPtr.Zero; } - + if (attr == null) { @@ -295,11 +295,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (ModuleObject)GetManagedObject(ob); - tp_clear(ob); - self.Dealloc(); + tp_clear(this.pyHandle); + base.Dealloc(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) @@ -345,7 +344,7 @@ protected override void OnSave(InterDomainContext context) if ((Runtime.PyDict_DelItemString(DictRef, pair.Key) == -1) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { - // Trying to remove a key that's not in the dictionary + // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. Runtime.PyErr_Clear(); } @@ -496,7 +495,7 @@ public static Assembly AddReference(string name) /// clr.GetClrType(IComparable) gives you the Type for IComparable, /// that you can e.g. perform reflection on. Similar to typeof(IComparable) in C# /// or clr.GetClrType(IComparable) in IronPython. - /// + /// /// /// /// The Type object diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 4e5a726bc..a73a9ae43 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -153,7 +153,6 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "__instancecheck__", "__subclasscheck__", "AddReference", - "FinalizeObject", "FindAssembly", "get_SuppressDocs", "get_SuppressOverloads", diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index e9fa91d3b..48fabca4a 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -58,14 +58,10 @@ public static IntPtr tp_repr(IntPtr op) return doc; } - /// - /// OverloadMapper dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Dealloc() { - var self = (OverloadMapper)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.target); + base.Dealloc(); } } } From 5bca333897bcb546a1fa079daf6f55559ca0a8f3 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 22:39:57 -0700 Subject: [PATCH 120/151] refactored tp_clear in ExtensionType and descendants into a virtual C# function Clear --- src/runtime/constructorbinding.cs | 14 ++++++-------- src/runtime/eventbinding.cs | 7 +++---- src/runtime/eventobject.cs | 10 ++++++++++ src/runtime/extensiontype.cs | 13 ++++++++++--- src/runtime/methodbinding.cs | 7 +++---- src/runtime/methodobject.cs | 9 ++++----- src/runtime/moduleobject.cs | 25 ++++++++++++------------- src/runtime/overload.cs | 6 ++++++ 8 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index ab4a7af0a..da35aafd1 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -155,11 +155,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) @@ -254,11 +253,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 60b9bba92..8fb0ce250 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -109,11 +109,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (EventBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.target); - return 0; + Runtime.Py_CLEAR(ref this.target); + base.Clear(); } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index e9bd98821..a0f06c375 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -207,6 +207,16 @@ protected override void Dealloc() } base.Dealloc(); } + + protected override void Clear() + { + if (this.unbound is not null) + { + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; + } + base.Clear(); + } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index db9eb0f72..38fe238de 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -62,6 +62,9 @@ protected virtual void Dealloc() this.FreeGCHandle(); } + /// DecRefs and nulls any fields pointing back to Python + protected virtual void Clear() { } + /// /// Type __setattr__ implementation. /// @@ -88,9 +91,6 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } - /// - /// Default dealloc implementation. - /// public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This @@ -99,6 +99,13 @@ public static void tp_dealloc(IntPtr ob) self?.Dealloc(); } + public static int tp_clear(IntPtr ob) + { + var self = (ExtensionType)GetManagedObject(ob); + self?.Clear(); + return 0; + } + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 783717189..a842dd308 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -241,11 +241,10 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - return 0; + this.ClearMembers(); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 5fa965f1b..c181c5df1 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -217,12 +217,11 @@ protected override void Dealloc() base.Dealloc(); } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - return 0; + this.ClearMembers(); + ClearObjectDict(this.pyHandle); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8892390c3..7d4992618 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -295,12 +295,6 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - protected override void Dealloc() - { - tp_clear(this.pyHandle); - base.Dealloc(); - } - public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ModuleObject)GetManagedObject(ob); @@ -314,17 +308,22 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - public static int tp_clear(IntPtr ob) + protected override void Dealloc() { - var self = (ModuleObject)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.dict); - ClearObjectDict(ob); - foreach (var attr in self.cache.Values) + tp_clear(this.pyHandle); + base.Dealloc(); + } + + protected override void Clear() + { + Runtime.Py_CLEAR(ref this.dict); + ClearObjectDict(this.pyHandle); + foreach (var attr in this.cache.Values) { Runtime.XDecref(attr.pyHandle); } - self.cache.Clear(); - return 0; + this.cache.Clear(); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 48fabca4a..c6c3158fb 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -63,5 +63,11 @@ protected override void Dealloc() Runtime.Py_CLEAR(ref this.target); base.Dealloc(); } + + protected override void Clear() + { + Runtime.Py_CLEAR(ref this.target); + base.Clear(); + } } } From 7271d88fae6d0776a66b92a45c796b518111fab4 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 22 May 2021 23:55:52 -0700 Subject: [PATCH 121/151] all Dealloc overrides simply duplicate Clear, so just call both from tp_dealloc and don't override Dealloc --- src/runtime/constructorbinding.cs | 12 ------------ src/runtime/eventbinding.cs | 6 ------ src/runtime/eventobject.cs | 10 ---------- src/runtime/extensiontype.cs | 15 ++++++++++++--- src/runtime/methodbinding.cs | 15 ++------------- src/runtime/methodobject.cs | 21 +++++---------------- src/runtime/moduleobject.cs | 6 ------ src/runtime/overload.cs | 6 ------ 8 files changed, 19 insertions(+), 72 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index da35aafd1..9ac1adc0f 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -149,12 +149,6 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.repr); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.repr); @@ -247,12 +241,6 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.repr); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.repr); diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 8fb0ce250..65c8fdccf 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -103,12 +103,6 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString(s); } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.target); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.target); diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index a0f06c375..941bbdf46 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -198,16 +198,6 @@ public static IntPtr tp_repr(IntPtr ob) } - protected override void Dealloc() - { - if (this.unbound is not null) - { - Runtime.XDecref(this.unbound.pyHandle); - this.unbound = null; - } - base.Dealloc(); - } - protected override void Clear() { if (this.unbound is not null) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 38fe238de..78df805ee 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -56,14 +56,22 @@ void SetupGc () protected virtual void Dealloc() { - ClearObjectDict(this.pyHandle); + var type = Runtime.PyObject_TYPE(this.ObjectReference); Runtime.PyObject_GC_Del(this.pyHandle); - // Not necessary for decref of `tpHandle`. + // Not necessary for decref of `tpHandle` - it is borrowed + this.FreeGCHandle(); + + // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc + Runtime.XDecref(type.DangerousGetAddress()); } /// DecRefs and nulls any fields pointing back to Python - protected virtual void Clear() { } + protected virtual void Clear() + { + ClearObjectDict(this.pyHandle); + // Not necessary for decref of `tpHandle` - it is borrowed + } /// /// Type __setattr__ implementation. @@ -96,6 +104,7 @@ public static void tp_dealloc(IntPtr ob) // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. var self = (ExtensionType)GetManagedObject(ob); + self?.Clear(); self?.Dealloc(); } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index a842dd308..c1e729f9e 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -229,21 +229,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref target); - Runtime.Py_CLEAR(ref targetType); - } - - protected override void Dealloc() - { - this.ClearMembers(); - base.Dealloc(); - } - protected override void Clear() { - this.ClearMembers(); + Runtime.Py_CLEAR(ref this.target); + Runtime.Py_CLEAR(ref this.targetType); base.Clear(); } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index c181c5df1..2787ec999 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -200,26 +200,15 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - private void ClearMembers() + protected override void Clear() { - Runtime.Py_CLEAR(ref doc); - if (unbound != null) + Runtime.Py_CLEAR(ref this.doc); + if (this.unbound != null) { - Runtime.XDecref(unbound.pyHandle); - unbound = null; + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; } - } - protected override void Dealloc() - { - this.ClearMembers(); - ClearObjectDict(this.pyHandle); - base.Dealloc(); - } - - protected override void Clear() - { - this.ClearMembers(); ClearObjectDict(this.pyHandle); base.Clear(); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 7d4992618..dfb6fdf55 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -308,12 +308,6 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - protected override void Dealloc() - { - tp_clear(this.pyHandle); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.dict); diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index c6c3158fb..8222dc136 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -58,12 +58,6 @@ public static IntPtr tp_repr(IntPtr op) return doc; } - protected override void Dealloc() - { - Runtime.Py_CLEAR(ref this.target); - base.Dealloc(); - } - protected override void Clear() { Runtime.Py_CLEAR(ref this.target); From 4f3f648d6a57f68ad99236baef26bd4afea56abc Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sat, 29 May 2021 14:49:28 -0700 Subject: [PATCH 122/151] fixup! merge latest master --- src/runtime/Python.Runtime.csproj | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index c158a19b2..0311dbf9a 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,9 +1,9 @@ - + netstandard2.0 AnyCPU 9.0 - Python.Runtime + Python.Runtime Python.Runtime pythonnet @@ -20,30 +20,30 @@ true snupkg - ..\pythonnet.snk + ..\pythonnet.snk true 1591;NU1701 True true - + ..\..\pythonnet\runtime false - + - + - + - + - - clr.py - - + + clr.py + + From c500a3929fb2cedd8c9727cde0e9ce7fdf0a4978 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sat, 29 May 2021 14:57:57 -0700 Subject: [PATCH 123/151] fixup! reworked `PythonException`: --- src/runtime/pythonexception.cs | 4 ++-- src/runtime/runtime.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 13316aef9..cca7c439f 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -354,8 +354,8 @@ public PythonException Clone() internal bool Is(IntPtr type) { return Runtime.PyErr_GivenExceptionMatches( - (Value ?? Type).Reference, - new BorrowedReference(type)) != 0; + given: (Value ?? Type).Reference, + typeOrTypes: new BorrowedReference(type)) != 0; } private static void CheckRuntimeIsRunning() diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5d6a3d192..6086135f5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2109,7 +2109,7 @@ internal static void PyErr_SetString(IntPtr ob, string message) internal static int PyErr_ExceptionMatches(IntPtr exception) => Delegates.PyErr_ExceptionMatches(exception); - internal static int PyErr_GivenExceptionMatches(BorrowedReference ob, BorrowedReference val) => Delegates.PyErr_GivenExceptionMatches(ob, val); + internal static int PyErr_GivenExceptionMatches(BorrowedReference given, BorrowedReference typeOrTypes) => Delegates.PyErr_GivenExceptionMatches(given, typeOrTypes); internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); From a321daa0147a49984a1df027df9d3f6104e77950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 15 Jan 2021 14:03:40 -0500 Subject: [PATCH 124/151] (WIP) modernize the import hook Implement a meta path loader instead --- src/runtime/assemblymanager.cs | 23 +++ src/runtime/extensiontype.cs | 2 +- src/runtime/importhook.cs | 273 +++++++------------------------ src/runtime/moduleobject.cs | 75 ++++++++- src/runtime/native/TypeOffset.cs | 1 + 5 files changed, 160 insertions(+), 214 deletions(-) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 0387d2dfc..fdde2aeb1 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -37,6 +37,9 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; internal static List pypath; + + // Triggered when a new namespace is added to the namespaces dictionary + public static event Action namespaceAdded; private AssemblyManager() { @@ -284,6 +287,17 @@ internal static void ScanAssembly(Assembly assembly) if (ns != null) { namespaces[ns].TryAdd(assembly, string.Empty); + try + { + namespaceAdded?.Invoke(ns); + } + catch (Exception e) + { + // For some reason, exceptions happening here does... nothing. + // Even System.AccessViolationExceptions gets ignored. + Console.WriteLine($"Namespace added callback failed with: {e}"); + throw; + } } if (ns != null && t.IsGenericTypeDefinition) @@ -312,6 +326,15 @@ public static bool IsValidNamespace(string name) return !string.IsNullOrEmpty(name) && namespaces.ContainsKey(name); } + /// + /// Returns an IEnumerable containing the namepsaces exported + /// by loaded assemblies in the current app domain. + /// + public static IEnumerable GetNamespaces () + { + return namespaces.Keys; + } + /// /// Returns list of assemblies that declare types in a given namespace /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 78df805ee..0ebd7ec4c 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -83,7 +83,7 @@ public static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) { message = "readonly attribute"; } - Exceptions.SetError(Exceptions.TypeError, message); + Exceptions.SetError(Exceptions.AttributeError, message); return -1; } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 9ac492d21..249fdebbe 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -15,57 +15,55 @@ internal static class ImportHook private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); - /// - /// Initialize just the __import__ hook itself. - /// - static void InitImport() - { - // We replace the built-in Python __import__ with our own: first - // look in CLR modules, then if we don't find any call the default - // Python __import__. - IntPtr builtins = Runtime.GetBuiltins(); - py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - PythonException.ThrowIfIsNull(py_import); - - hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr); - PythonException.ThrowIfIsNotZero(res); - - Runtime.XDecref(builtins); - } - - /// - /// Restore the __import__ hook. - /// - static void RestoreImport() - { - IntPtr builtins = Runtime.GetBuiltins(); - - IntPtr existing = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - Runtime.XDecref(existing); - if (existing != hook.ptr) - { - throw new NotSupportedException("Unable to restore original __import__."); - } - - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); - PythonException.ThrowIfIsNotZero(res); - Runtime.XDecref(py_import); - py_import = IntPtr.Zero; - - hook.Release(); - hook = null; - - Runtime.XDecref(builtins); - } + private const string LoaderCode = @" +import importlib.abc +import sys + +class DotNetLoader(importlib.abc.Loader): + + def __init__(self): + super(DotNetLoader, self).__init__() + + @classmethod + def exec_module(klass, mod): + # this method is needed to mark this + # loader as a non-legacy loader. + pass + + @classmethod + def create_module(klass, spec): + import clr + a = clr._LoadClrModule(spec) + #mod = getattr(clr, '__imported') + print(a) + #print(mod) + #print(mod is a) + #delattr(clr, '__imported') + return a + +class DotNetFinder(importlib.abc.MetaPathFinder): + + def __init__(self): + print('DotNetFinder init') + super(DotNetFinder, self).__init__() + + @classmethod + def find_spec(klass, fullname, paths=None, target=None): + import clr + # print(clr._availableNamespaces) + if (hasattr(clr, '_availableNamespaces') and fullname in clr._availableNamespaces): + #if (clr._NamespaceLoaded(fullname)): + return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) + return None + +sys.meta_path.append(DotNetFinder()) + "; /// /// Initialization performed on startup of the Python runtime. /// internal static unsafe void Initialize() { - InitImport(); - // Initialize the clr module and tell Python about it. root = new CLRModule(); @@ -80,6 +78,9 @@ internal static unsafe void Initialize() BorrowedReference dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + + // Add/create the MetaPathLoader + PythonEngine.Exec(LoaderCode); } @@ -93,8 +94,7 @@ internal static void Shutdown() return; } - RestoreImport(); - + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; @@ -115,7 +115,6 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) internal static void RestoreRuntimeData(RuntimeDataStorage storage) { - InitImport(); storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); @@ -136,41 +135,6 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa Runtime.PyDict_Update(py_mod_dict, clr_dict); } - // find any items from the from list and get them from the root if they're not - // already in the module dictionary - if (fromList != null) - { - if (Runtime.PyTuple_Check(fromList)) - { - using var mod_dict = new PyDict(py_mod_dict); - using var from = new PyTuple(fromList); - foreach (PyObject item in from) - { - if (mod_dict.HasKey(item)) - { - continue; - } - - var s = item.AsManagedObject(typeof(string)) as string; - if (s == null) - { - continue; - } - - ManagedType attr = root.GetAttribute(s, true); - if (attr == null) - { - continue; - } - - Runtime.XIncref(attr.pyHandle); - using (var obj = new PyObject(attr.pyHandle)) - { - mod_dict.SetItem(s, obj); - } - } - } - } Runtime.XIncref(py_clr_module); return NewReference.DangerousFromPointer(py_clr_module); } @@ -178,124 +142,15 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa /// /// The actual import hook that ties Python to the managed world. /// - public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) + public static ModuleObject __import__(string modname) { - var args = new BorrowedReference(argsRaw); - - // Replacement for the builtin __import__. The original import - // hook is saved as this.py_import. This version handles CLR - // import and defers to the normal builtin for everything else. - - var num_args = Runtime.PyTuple_Size(args); - if (num_args < 1) - { - return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); - } - - BorrowedReference py_mod_name = Runtime.PyTuple_GetItem(args, 0); - if (py_mod_name.IsNull || - !Runtime.IsStringType(py_mod_name)) - { - return Exceptions.RaiseTypeError("string expected"); - } - - // Check whether the import is of the form 'from x import y'. - // This determines whether we return the head or tail module. - - BorrowedReference fromList = default; - var fromlist = false; - if (num_args >= 4) - { - fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != null && - Runtime.PyObject_IsTrue(fromList) == 1) - { - fromlist = true; - } - } - - string mod_name = Runtime.GetManagedString(py_mod_name); - // Check these BEFORE the built-in import runs; may as well - // do the Incref()ed return here, since we've already found - // the module. - if (mod_name == "clr") - { - NewReference clr_module = GetCLRModule(fromList); - if (!clr_module.IsNull()) - { - BorrowedReference sys_modules = Runtime.PyImport_GetModuleDict(); - if (!sys_modules.IsNull) - { - Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); - } - } - return clr_module.DangerousMoveToPointerOrNull(); - } - - string realname = mod_name; - - // 2010-08-15: Always seemed smart to let python try first... - // This shaves off a few tenths of a second on test_module.py - // and works around a quirk where 'sys' is found by the - // LoadImplicit() deprecation logic. - // Turns out that the AssemblyManager.ResolveHandler() checks to see if any - // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very - // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args.DangerousGetAddress(), kw); - if (res != IntPtr.Zero) - { - // There was no error. - if (fromlist && IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); - } - return res; - } - // There was an error - if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) - { - // and it was NOT an ImportError; bail out here. - return IntPtr.Zero; - } - - if (mod_name == string.Empty) - { - // Most likely a missing relative import. - // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: - // from . import _html5lib - // We don't support them anyway - return IntPtr.Zero; - } - // Save the exception - var originalException = PythonException.FetchCurrentRaw(); + // Console.WriteLine("Import hook"); + string realname = modname; string[] names = realname.Split('.'); - // See if sys.modules for this interpreter already has the - // requested module. If so, just return the existing module. - BorrowedReference modules = Runtime.PyImport_GetModuleDict(); - BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name); - - if (module != null) - { - if (fromlist) - { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } - return new NewReference(module).DangerousMoveToPointer(); - } - - module = Runtime.PyDict_GetItemString(modules, names[0]); - return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); - } - Exceptions.Clear(); - - // Traverse the qualified module name to get the named module - // and place references in sys.modules as we go. Note that if + // Traverse the qualified module name to get the named module. + // Note that if // we are running in interactive mode we pre-load the names in // each module, which is often useful for introspection. If we // are not interactive, we stick to just-in-time creation of @@ -304,17 +159,21 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) // enable preloading in a non-interactive python processing by // setting clr.preload = True - ModuleObject head = mod_name == realname ? null : root; + ModuleObject head = null; ModuleObject tail = root; root.InitializePreload(); + // root.LoadNames(); foreach (string name in names) { ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - originalException.Restore(); - return IntPtr.Zero; + // originalException.Restore(); + // Exceptions.SetError(Exceptions.ImportError, ""); + // throw new PythonException(); + // TODO: set exception + return null; } if (head == null) { @@ -325,21 +184,11 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) { tail.LoadNames(); } - - // Add the module to sys.modules - Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); } { - var mod = fromlist ? tail : head; - - if (fromlist && IsLoadAll(fromList)) - { - mod.LoadNames(); - } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + Runtime.XIncref(tail.pyHandle); + return tail; } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index dfb6fdf55..dd4f7f625 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -20,6 +20,7 @@ internal class ModuleObject : ExtensionType internal IntPtr dict; internal BorrowedReference DictRef => new BorrowedReference(dict); protected string _namespace; + private IntPtr __all__ = IntPtr.Zero; public ModuleObject(string name) { @@ -181,7 +182,23 @@ public void LoadNames() { continue; } - GetAttribute(name, true); + + if(GetAttribute(name, true) != null) + { + // if it's a valid attribute, add it to __all__ + var pyname = Runtime.PyString_FromString(name); + try + { + if (Runtime.PyList_Append(new BorrowedReference(__all__), pyname) != 0) + { + throw new PythonException(); + } + } + finally + { + Runtime.XDecref(pyname); + } + } } } @@ -263,6 +280,13 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } + if (name == "__all__") + { + self.LoadNames(); + Runtime.XIncref(self.__all__); + return self.__all__; + } + ManagedType attr = null; try @@ -320,6 +344,32 @@ protected override void Clear() base.Clear(); } + /// + /// Override the setattr implementation. + /// This is needed because the import mechanics need + /// to set a few attributes + /// + [ForbidPythonThreads] + public new static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) + { + var self = (ModuleObject)ManagedType.GetManagedObject(ob); + var dict = self.dict; + + var current = Runtime.PyDict_GetItem(dict, key); + if (current == val) + { + return 0; + } + else if (ManagedType.GetManagedObject(current) != null) + { + var message = "Can't override a .NET object"; + Exceptions.SetError(Exceptions.AttributeError, message); + return -1; + } + + return Runtime.PyDict_SetItem(dict, key, val); + } + protected override void OnSave(InterDomainContext context) { base.OnSave(context); @@ -526,5 +576,28 @@ public static string[] ListAssemblies(bool verbose) } return names; } + + [ModuleFunction] + public static int _AtExit() + { + return Runtime.AtExit(); + } + + + [ModuleFunction] + [ForbidPythonThreads] + public static PyObject _LoadClrModule(PyObject spec) + { + var mod = ImportHook.__import__(spec.GetAttr("name").ToString()); + if (mod == null) + { + // __import__ sets the exception.? + Console.WriteLine("NULL module"); + return Runtime.None; + } + // We can't return directly a ModuleObject, because the tpHandle is + // not set, but we can return a PyObject. + return new PyObject(mod.pyHandle); + } } } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index a73a9ae43..d9c3ee52c 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -160,6 +160,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "getPreload", "Initialize", "ListAssemblies", + "_LoadClrModule", "Release", "Reset", "set_SuppressDocs", From 279b53546d358582ac9b22ef12b5ad74ba35b064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 15 Jan 2021 15:27:48 -0500 Subject: [PATCH 125/151] Add the loaded namespaces tracking --- src/runtime/importhook.cs | 114 ++++++++++++++++++++++++++++++------ src/runtime/moduleobject.cs | 10 +--- 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 249fdebbe..86c2cbac8 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Runtime.InteropServices; namespace Python.Runtime @@ -58,6 +57,7 @@ import clr sys.meta_path.append(DotNetFinder()) "; + const string availableNsKey = "_availableNamespaces"; /// /// Initialization performed on startup of the Python runtime. @@ -80,6 +80,7 @@ internal static unsafe void Initialize() Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); // Add/create the MetaPathLoader + SetupNamespaceTracking(); PythonEngine.Exec(LoaderCode); } @@ -98,6 +99,7 @@ internal static void Shutdown() Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; + TeardownNameSpaceTracking(); Runtime.XDecref(root.pyHandle); root = null; CLRModule.Reset(); @@ -118,12 +120,90 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + SetupNamespaceTracking(); } + static void SetupNamespaceTracking () + { + var newset = Runtime.PySet_New(IntPtr.Zero); + try + { + foreach (var ns in AssemblyManager.GetNamespaces()) + { + var pyNs = Runtime.PyString_FromString(ns); + try + { + if(Runtime.PySet_Add(newset, pyNs) != 0) + { + throw new PythonException(); + } + } + finally + { + Runtime.XDecref(pyNs); + } + } + + if(Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset) != 0) + { + throw new PythonException(); + } + } + finally + { + Runtime.XDecref(newset); + } + + AssemblyManager.namespaceAdded += OnNamespaceAdded; + PythonEngine.AddShutdownHandler(()=>AssemblyManager.namespaceAdded -= OnNamespaceAdded); + } + + static void TeardownNameSpaceTracking() + { + AssemblyManager.namespaceAdded -= OnNamespaceAdded; + // If the C# runtime isn't loaded, then there is no namespaces available + if ((Runtime.PyDict_DelItemString(root.dict, availableNsKey) != 0) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + } + + static void OnNamespaceAdded (string name) + { + using(Py.GIL()) + { + var pyNs = Runtime.PyString_FromString(name); + try + { + var nsSet = Runtime.PyDict_GetItemString(root.dict, availableNsKey); + if (nsSet != IntPtr.Zero) + { + if(Runtime.PySet_Add(nsSet, pyNs) != 0) + { + throw new PythonException(); + } + } + } + finally + { + Runtime.XDecref(pyNs); + } + } + } + + /// - /// Return the clr python module (new reference) + /// Because we use a proxy module for the clr module, we somtimes need + /// to force the py_clr_module to sync with the actual clr module's dict. /// - public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) + internal static void UpdateCLRModuleDict() { root.InitializePreload(); @@ -134,21 +214,23 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa { Runtime.PyDict_Update(py_mod_dict, clr_dict); } + } + /// + /// Return the clr python module (new reference) + /// + public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) + { + UpdateCLRModuleDict(); Runtime.XIncref(py_clr_module); return NewReference.DangerousFromPointer(py_clr_module); } /// - /// The actual import hook that ties Python to the managed world. + /// The hook to import a CLR module into Python /// public static ModuleObject __import__(string modname) { - // Console.WriteLine("Import hook"); - - string realname = modname; - string[] names = realname.Split('.'); - // Traverse the qualified module name to get the named module. // Note that if // we are running in interactive mode we pre-load the names in @@ -162,18 +244,15 @@ public static ModuleObject __import__(string modname) ModuleObject head = null; ModuleObject tail = root; root.InitializePreload(); - // root.LoadNames(); + string[] names = modname.Split('.'); foreach (string name in names) { ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - // originalException.Restore(); - // Exceptions.SetError(Exceptions.ImportError, ""); - // throw new PythonException(); - // TODO: set exception - return null; + Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); + throw new PythonException(); } if (head == null) { @@ -186,10 +265,7 @@ public static ModuleObject __import__(string modname) } } - { - Runtime.XIncref(tail.pyHandle); - return tail; - } + return tail; } private static bool IsLoadAll(BorrowedReference fromList) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index dd4f7f625..1b80390a5 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -529,7 +529,8 @@ public static Assembly AddReference(string name) { throw new FileNotFoundException($"Unable to find assembly '{name}'."); } - + // Classes that are not in a namespace needs an extra nudge to be found. + ImportHook.UpdateCLRModuleDict(); return assembly; } @@ -583,18 +584,11 @@ public static int _AtExit() return Runtime.AtExit(); } - [ModuleFunction] [ForbidPythonThreads] public static PyObject _LoadClrModule(PyObject spec) { var mod = ImportHook.__import__(spec.GetAttr("name").ToString()); - if (mod == null) - { - // __import__ sets the exception.? - Console.WriteLine("NULL module"); - return Runtime.None; - } // We can't return directly a ModuleObject, because the tpHandle is // not set, but we can return a PyObject. return new PyObject(mod.pyHandle); From f92e95b39dc67979dcc17497edfe99873a17bbce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 15 Jan 2021 15:45:46 -0500 Subject: [PATCH 126/151] cleanup and changelog entry --- CHANGELOG.md | 2 ++ src/runtime/importhook.cs | 15 ++------------- src/runtime/moduleobject.cs | 6 +++++- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5871e7ffb..326c229ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - .NET and Python exceptions are preserved when crossing Python/.NET boundary - BREAKING: custom encoders are no longer called for instances of `System.Type` - `PythonException.Restore` no longer clears `PythonException` instance. +- Replaced the old `__import__` hook hack with a PEP302-style Meta Path Loader ### Fixed @@ -72,6 +73,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - Providing an invalid type parameter to a generic type or method produces a helpful Python error - Empty parameter names (as can be generated from F#) do not cause crashes + ### Removed - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 86c2cbac8..02c9dc364 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -8,9 +8,7 @@ namespace Python.Runtime /// internal static class ImportHook { - private static IntPtr py_import; private static CLRModule root; - private static MethodWrapper hook; private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); @@ -25,20 +23,13 @@ def __init__(self): @classmethod def exec_module(klass, mod): - # this method is needed to mark this - # loader as a non-legacy loader. + # This method needs to exist. pass @classmethod def create_module(klass, spec): import clr - a = clr._LoadClrModule(spec) - #mod = getattr(clr, '__imported') - print(a) - #print(mod) - #print(mod is a) - #delattr(clr, '__imported') - return a + return clr._LoadClrModule(spec) class DotNetFinder(importlib.abc.MetaPathFinder): @@ -49,9 +40,7 @@ def __init__(self): @classmethod def find_spec(klass, fullname, paths=None, target=None): import clr - # print(clr._availableNamespaces) if (hasattr(clr, '_availableNamespaces') and fullname in clr._availableNamespaces): - #if (clr._NamespaceLoaded(fullname)): return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) return None diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 1b80390a5..a319867d6 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -588,7 +588,11 @@ public static int _AtExit() [ForbidPythonThreads] public static PyObject _LoadClrModule(PyObject spec) { - var mod = ImportHook.__import__(spec.GetAttr("name").ToString()); + ModuleObject mod = null; + using (var modname = spec.GetAttr("name")) + { + mod = ImportHook.__import__(modname.ToString()); + } // We can't return directly a ModuleObject, because the tpHandle is // not set, but we can return a PyObject. return new PyObject(mod.pyHandle); From afffc187515fa7c8e725f2e0cc592cc04966ef7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 18 Jan 2021 16:17:25 -0500 Subject: [PATCH 127/151] Fix a bug where clr wasn't in sys.modules after reload --- src/runtime/importhook.cs | 3 ++ tests/domain_tests/TestRunner.cs | 40 ++++++++++++++++++++++++ tests/domain_tests/test_domain_reload.py | 4 +++ 3 files changed, 47 insertions(+) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 02c9dc364..5af702217 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -109,6 +109,9 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + IntPtr dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); + Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); SetupNamespaceTracking(); } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index a21297829..83ae1654d 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -1092,6 +1092,46 @@ assert sys.my_obj is not None foo = sys.my_obj.Inner() print(foo) + ", + }, + new TestCase + { + // The C# code for this test doesn't matter. + Name = "import_after_reload", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Cls + { + } + } + }", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + + +def after_reload(): + assert 'System' in sys.modules + assert 'clr' in sys.modules + import clr + import System + ", }, }; diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index a7cd2fa4d..71b1e348e 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -88,3 +88,7 @@ def test_in_to_ref_param(): def test_nested_type(): _run_test("nested_type") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_import_after_reload(): + _run_test("import_after_reload") \ No newline at end of file From d821c0f7094f92508babc4dc7168f133f12894cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 19 Jan 2021 11:38:29 -0500 Subject: [PATCH 128/151] Further refinements to setattr logic on ModuleObjects --- src/runtime/moduleobject.cs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index a319867d6..eadfeadfb 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -22,6 +22,11 @@ internal class ModuleObject : ExtensionType protected string _namespace; private IntPtr __all__ = IntPtr.Zero; + // Attributes to be set on the module according to PEP302 and 451 + // by the import machinery. + static readonly HashSet settableAttributes = + new HashSet {"__spec__", "__file__", "__name__", "__path__", "__loader__", "__package__"}; + public ModuleObject(string name) { if (name == string.Empty) @@ -352,22 +357,15 @@ protected override void Clear() [ForbidPythonThreads] public new static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) { - var self = (ModuleObject)ManagedType.GetManagedObject(ob); - var dict = self.dict; - - var current = Runtime.PyDict_GetItem(dict, key); - if (current == val) - { - return 0; - } - else if (ManagedType.GetManagedObject(current) != null) + var managedKey = Runtime.GetManagedString(key); + if ((settableAttributes.Contains(managedKey)) || + (ManagedType.GetManagedObject(val)?.GetType() == typeof(ModuleObject)) ) { - var message = "Can't override a .NET object"; - Exceptions.SetError(Exceptions.AttributeError, message); - return -1; + var self = (ModuleObject)ManagedType.GetManagedObject(ob); + return Runtime.PyDict_SetItem(self.dict, key, val); } - return Runtime.PyDict_SetItem(dict, key, val); + return ExtensionType.tp_setattro(ob, key, val); } protected override void OnSave(InterDomainContext context) From e469a8a73cfc0870cb56ba0368cff44c3938dd24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 20 Jan 2021 13:26:23 -0500 Subject: [PATCH 129/151] fixups, add docs --- CHANGELOG.md | 1 - src/runtime/importhook.cs | 11 ++++++++++- src/runtime/moduleobject.cs | 9 +++++++++ tests/domain_tests/TestRunner.cs | 25 ++++--------------------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 326c229ea..f72016742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,6 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - Providing an invalid type parameter to a generic type or method produces a helpful Python error - Empty parameter names (as can be generated from F#) do not cause crashes - ### Removed - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 5af702217..335ff1af8 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -34,7 +34,6 @@ import clr class DotNetFinder(importlib.abc.MetaPathFinder): def __init__(self): - print('DotNetFinder init') super(DotNetFinder, self).__init__() @classmethod @@ -115,6 +114,12 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) SetupNamespaceTracking(); } + /// + /// Sets up the tracking of loaded namespaces. This makes available to + /// Python, as a Python object, the loaded namespaces. The set of loaded + /// namespaces is used during the import to verify if we can import a + /// CLR assembly as a module or not. The set is stored on the clr module. + /// static void SetupNamespaceTracking () { var newset = Runtime.PySet_New(IntPtr.Zero); @@ -150,6 +155,10 @@ static void SetupNamespaceTracking () PythonEngine.AddShutdownHandler(()=>AssemblyManager.namespaceAdded -= OnNamespaceAdded); } + /// + /// Removes the set of available namespaces from the clr module and + /// removes the callback on the OnNamespaceAdded event. + /// static void TeardownNameSpaceTracking() { AssemblyManager.namespaceAdded -= OnNamespaceAdded; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index eadfeadfb..9eb1d997d 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -582,6 +582,15 @@ public static int _AtExit() return Runtime.AtExit(); } + + /// + /// Note: This should *not* be called directly. + /// The function that get/import a CLR assembly as a python module. + /// This function should only be called by the import machinery as seen + /// in importhook.cs + /// + /// A ModuleSpec Python object + /// A new reference to the imported module, as a PyObject. [ModuleFunction] [ForbidPythonThreads] public static PyObject _LoadClrModule(PyObject spec) diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index 83ae1654d..716fe079b 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -1096,28 +1096,11 @@ assert sys.my_obj is not None }, new TestCase { - // The C# code for this test doesn't matter. + // The C# code for this test doesn't matter; we're testing + // that the import hook behaves properly after a domain reload Name = "import_after_reload", - DotNetBefore = @" - namespace TestNamespace - { - [System.Serializable] - public class Cls - { - } - }", - DotNetAfter = @" - namespace TestNamespace - { - [System.Serializable] - public class WithNestedType - { - [System.Serializable] - public class Cls - { - } - } - }", + DotNetBefore = "", + DotNetAfter = "", PythonCode = @" import sys From 685b972001d2186e60d41f7a42959f74e6e8a5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 4 Mar 2021 13:17:41 -0500 Subject: [PATCH 130/151] merge fixup --- src/embed_tests/TestPythonException.cs | 1 + src/runtime/importhook.cs | 35 +++++++++++--------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 0763bfb34..a4b28906c 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -178,6 +178,7 @@ public void TestPythonException_Normalize_ThrowsWhenErrorSet() var pythonException = PythonException.FetchCurrentRaw(); Exceptions.SetError(Exceptions.TypeError, "Another error"); Assert.Throws(() => pythonException.Normalize()); + Exceptions.Clear(); } } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 335ff1af8..948af56e0 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -108,9 +108,9 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); - IntPtr dict = Runtime.PyImport_GetModuleDict(); - Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); - Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "CLR", py_clr_module); + Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "clr", py_clr_module); SetupNamespaceTracking(); } @@ -122,7 +122,7 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) /// static void SetupNamespaceTracking () { - var newset = Runtime.PySet_New(IntPtr.Zero); + var newset = Runtime.PySet_New(new BorrowedReference(IntPtr.Zero)); try { foreach (var ns in AssemblyManager.GetNamespaces()) @@ -130,7 +130,7 @@ static void SetupNamespaceTracking () var pyNs = Runtime.PyString_FromString(ns); try { - if(Runtime.PySet_Add(newset, pyNs) != 0) + if(Runtime.PySet_Add(newset, new BorrowedReference(pyNs)) != 0) { throw new PythonException(); } @@ -141,14 +141,14 @@ static void SetupNamespaceTracking () } } - if(Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset) != 0) + if(Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset.DangerousGetAddress()) != 0) { throw new PythonException(); } } finally { - Runtime.XDecref(newset); + newset.Dispose(); } AssemblyManager.namespaceAdded += OnNamespaceAdded; @@ -163,17 +163,13 @@ static void TeardownNameSpaceTracking() { AssemblyManager.namespaceAdded -= OnNamespaceAdded; // If the C# runtime isn't loaded, then there is no namespaces available - if ((Runtime.PyDict_DelItemString(root.dict, availableNsKey) != 0) && + if ((Runtime.PyDict_DelItemString(new BorrowedReference(root.dict), availableNsKey) != 0) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. Runtime.PyErr_Clear(); } - else if (Exceptions.ErrorOccurred()) - { - throw new PythonException(); - } } static void OnNamespaceAdded (string name) @@ -183,10 +179,10 @@ static void OnNamespaceAdded (string name) var pyNs = Runtime.PyString_FromString(name); try { - var nsSet = Runtime.PyDict_GetItemString(root.dict, availableNsKey); - if (nsSet != IntPtr.Zero) + var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); + if (!nsSet.IsNull) { - if(Runtime.PySet_Add(nsSet, pyNs) != 0) + if(Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) { throw new PythonException(); } @@ -211,16 +207,15 @@ internal static void UpdateCLRModuleDict() // update the module dictionary with the contents of the root dictionary root.LoadNames(); BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - using (var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference)) - { - Runtime.PyDict_Update(py_mod_dict, clr_dict); - } + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); + + Runtime.PyDict_Update(py_mod_dict, clr_dict); } /// /// Return the clr python module (new reference) /// - public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) + public static unsafe NewReference GetCLRModule() { UpdateCLRModuleDict(); Runtime.XIncref(py_clr_module); From be8136448e1220f2a3ea93f7243c2f4a8eaa6237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 10 Mar 2021 12:57:39 -0500 Subject: [PATCH 131/151] (WIP) import hook in the pytohn module --- pythonnet/util/__init__.py | 2 +- pythonnet/util/import_hook.py | 33 +++++++++++++++++++++++++++++++++ src/runtime/importhook.cs | 4 ++-- src/runtime/moduleobject.cs | 2 +- src/runtime/runtime.cs | 20 ++++++++++---------- 5 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 pythonnet/util/import_hook.py diff --git a/pythonnet/util/__init__.py b/pythonnet/util/__init__.py index 75d4bad8c..e78fbafe6 100644 --- a/pythonnet/util/__init__.py +++ b/pythonnet/util/__init__.py @@ -1 +1 @@ -from .find_libpython import find_libpython +from ..find_libpython import find_libpython diff --git a/pythonnet/util/import_hook.py b/pythonnet/util/import_hook.py new file mode 100644 index 000000000..fc8692cad --- /dev/null +++ b/pythonnet/util/import_hook.py @@ -0,0 +1,33 @@ +import importlib.abc +import sys + +class DotNetLoader(importlib.abc.Loader): + + def __init__(self): + super(DotNetLoader, self).__init__() + + @classmethod + def exec_module(klass, mod): + # This method needs to exist. + pass + + @classmethod + def create_module(klass, spec): + import clr + return clr._LoadClrModule(spec) + +class DotNetFinder(importlib.abc.MetaPathFinder): + + def __init__(self): + super(DotNetFinder, self).__init__() + + @classmethod + def find_spec(klass, fullname, paths=None, target=None): + import clr + if (hasattr(clr, '_availableNamespaces') and fullname in clr._availableNamespaces): + return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) + return None + + +def init_import_hook(): + sys.meta_path.append(DotNetFinder()) \ No newline at end of file diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 948af56e0..4d776123c 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -69,7 +69,7 @@ internal static unsafe void Initialize() // Add/create the MetaPathLoader SetupNamespaceTracking(); - PythonEngine.Exec(LoaderCode); + PythonEngine.Exec("import pythonnet.util.import_hook;pythonnet.util.import_hook.init_import_hook()"); } @@ -225,7 +225,7 @@ public static unsafe NewReference GetCLRModule() /// /// The hook to import a CLR module into Python /// - public static ModuleObject __import__(string modname) + public static ModuleObject Import(string modname) { // Traverse the qualified module name to get the named module. // Note that if diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 9eb1d997d..e710b8ba9 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -598,7 +598,7 @@ public static PyObject _LoadClrModule(PyObject spec) ModuleObject mod = null; using (var modname = spec.GetAttr("name")) { - mod = ImportHook.__import__(modname.ToString()); + mod = ImportHook.Import(modname.ToString()); } // We can't return directly a ModuleObject, because the tpHandle is // not set, but we can return a PyObject. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 789b71f3e..9f8c87030 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -153,6 +153,16 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd ClassDerivedObject.Reset(); TypeManager.Initialize(); + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path").DangerousGetAddress(); + IntPtr item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(new BorrowedReference(path), item); + } + XDecref(item); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); OperatorMethod.Initialize(); @@ -167,16 +177,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } Exceptions.Initialize(); - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - IntPtr path = PySys_GetObject("path").DangerousGetAddress(); - IntPtr item = PyString_FromString(rtdir); - if (PySequence_Contains(path, item) == 0) - { - PyList_Append(new BorrowedReference(path), item); - } - XDecref(item); AssemblyManager.UpdatePath(); } From 73958ed239b21cc71c0206c3e5ffcc45ed7aa182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 12 Apr 2021 11:14:04 -0400 Subject: [PATCH 132/151] Revert "(WIP) import hook in the pytohn module" This reverts commit 4684523c076f9815203ae26ff8a6418158315fc3. --- pythonnet/util/__init__.py | 2 +- pythonnet/util/import_hook.py | 33 --------------------------------- src/runtime/importhook.cs | 4 ++-- src/runtime/moduleobject.cs | 2 +- src/runtime/runtime.cs | 20 ++++++++++---------- 5 files changed, 14 insertions(+), 47 deletions(-) delete mode 100644 pythonnet/util/import_hook.py diff --git a/pythonnet/util/__init__.py b/pythonnet/util/__init__.py index e78fbafe6..75d4bad8c 100644 --- a/pythonnet/util/__init__.py +++ b/pythonnet/util/__init__.py @@ -1 +1 @@ -from ..find_libpython import find_libpython +from .find_libpython import find_libpython diff --git a/pythonnet/util/import_hook.py b/pythonnet/util/import_hook.py deleted file mode 100644 index fc8692cad..000000000 --- a/pythonnet/util/import_hook.py +++ /dev/null @@ -1,33 +0,0 @@ -import importlib.abc -import sys - -class DotNetLoader(importlib.abc.Loader): - - def __init__(self): - super(DotNetLoader, self).__init__() - - @classmethod - def exec_module(klass, mod): - # This method needs to exist. - pass - - @classmethod - def create_module(klass, spec): - import clr - return clr._LoadClrModule(spec) - -class DotNetFinder(importlib.abc.MetaPathFinder): - - def __init__(self): - super(DotNetFinder, self).__init__() - - @classmethod - def find_spec(klass, fullname, paths=None, target=None): - import clr - if (hasattr(clr, '_availableNamespaces') and fullname in clr._availableNamespaces): - return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) - return None - - -def init_import_hook(): - sys.meta_path.append(DotNetFinder()) \ No newline at end of file diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 4d776123c..948af56e0 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -69,7 +69,7 @@ internal static unsafe void Initialize() // Add/create the MetaPathLoader SetupNamespaceTracking(); - PythonEngine.Exec("import pythonnet.util.import_hook;pythonnet.util.import_hook.init_import_hook()"); + PythonEngine.Exec(LoaderCode); } @@ -225,7 +225,7 @@ public static unsafe NewReference GetCLRModule() /// /// The hook to import a CLR module into Python /// - public static ModuleObject Import(string modname) + public static ModuleObject __import__(string modname) { // Traverse the qualified module name to get the named module. // Note that if diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index e710b8ba9..9eb1d997d 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -598,7 +598,7 @@ public static PyObject _LoadClrModule(PyObject spec) ModuleObject mod = null; using (var modname = spec.GetAttr("name")) { - mod = ImportHook.Import(modname.ToString()); + mod = ImportHook.__import__(modname.ToString()); } // We can't return directly a ModuleObject, because the tpHandle is // not set, but we can return a PyObject. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9f8c87030..789b71f3e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -153,16 +153,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd ClassDerivedObject.Reset(); TypeManager.Initialize(); - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - IntPtr path = PySys_GetObject("path").DangerousGetAddress(); - IntPtr item = PyString_FromString(rtdir); - if (PySequence_Contains(path, item) == 0) - { - PyList_Append(new BorrowedReference(path), item); - } - XDecref(item); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); OperatorMethod.Initialize(); @@ -177,6 +167,16 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } Exceptions.Initialize(); + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path").DangerousGetAddress(); + IntPtr item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(new BorrowedReference(path), item); + } + XDecref(item); AssemblyManager.UpdatePath(); } From e71a0ef90414bd2cde21901ed3956251fccc4753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 12 Apr 2021 15:39:12 -0400 Subject: [PATCH 133/151] Import hook as a module, take 2 --- src/runtime/importhook.cs | 93 +++++++++++++++++++++++--------- src/runtime/moduleobject.cs | 4 +- src/runtime/native/TypeOffset.cs | 2 +- 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 948af56e0..df1ae253e 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -19,7 +19,7 @@ import sys class DotNetLoader(importlib.abc.Loader): def __init__(self): - super(DotNetLoader, self).__init__() + super().__init__() @classmethod def exec_module(klass, mod): @@ -29,21 +29,19 @@ def exec_module(klass, mod): @classmethod def create_module(klass, spec): import clr - return clr._LoadClrModule(spec) + return clr._load_clr_module(spec) class DotNetFinder(importlib.abc.MetaPathFinder): def __init__(self): - super(DotNetFinder, self).__init__() + super().__init__() @classmethod def find_spec(klass, fullname, paths=None, target=None): import clr - if (hasattr(clr, '_availableNamespaces') and fullname in clr._availableNamespaces): + if clr._availableNamespaces and fullname in clr._availableNamespaces: return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) return None - -sys.meta_path.append(DotNetFinder()) "; const string availableNsKey = "_availableNamespaces"; @@ -69,7 +67,7 @@ internal static unsafe void Initialize() // Add/create the MetaPathLoader SetupNamespaceTracking(); - PythonEngine.Exec(LoaderCode); + SetupImportHook(); } @@ -114,13 +112,66 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) SetupNamespaceTracking(); } + static void SetupImportHook() + { + // Create the import hook module + var import_hook_module_def = ModuleDefOffset.AllocModuleDef("clr.loader"); + var import_hook_module = Runtime.PyModule_Create2(import_hook_module_def, 3); + + // Run the python code to create the module's classes. + var mod_dict = Runtime.PyModule_GetDict(new BorrowedReference(import_hook_module)); + var builtins = Runtime.PyEval_GetBuiltins(); + var exec = Runtime.PyDict_GetItemString(builtins, "exec"); + using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); + + var codeStr = Runtime.PyString_FromString(LoaderCode); + Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 0, codeStr); + // PyTuple_SetItem steals a reference, mod_dict is borrowed. + Runtime.XIncref(mod_dict.DangerousGetAddress()); + Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 1, mod_dict.DangerousGetAddress()); + Runtime.PyObject_Call(exec.DangerousGetAddress(), args.DangerousGetAddress(), IntPtr.Zero); + + var loader = Runtime.PyDict_GetItemString(mod_dict, "DotNetLoader").DangerousGetAddressOrNull(); + Runtime.XIncref(loader); + + // Add the classes to the module + // PyModule_AddObject steals a reference only on success + if (Runtime.PyModule_AddObject(import_hook_module, "DotNetLoader", loader) != 0) + { + Runtime.XDecref(loader); + throw new PythonException(); + } + + var finder = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder").DangerousGetAddressOrNull(); + Runtime.XIncref(finder); + if (Runtime.PyModule_AddObject(import_hook_module, "DotNetFinder", finder) != 0) + { + Runtime.XDecref(finder); + throw new PythonException(); + } + + // Set as a sub-module of clr. + Runtime.XIncref(import_hook_module); + if(Runtime.PyModule_AddObject(py_clr_module, "loader", import_hook_module) != 0) + { + Runtime.XDecref(import_hook_module); + throw new PythonException(); + } + + // Finally, add the hook to the meta path + var finder_inst = Runtime.PyDict_GetItemString(mod_dict, "finder_inst").DangerousGetAddressOrNull(); + Runtime.XIncref(finder); + var metapath = Runtime.PySys_GetObject("meta_path"); + Runtime.PyList_Append(metapath, finder); + } + /// /// Sets up the tracking of loaded namespaces. This makes available to /// Python, as a Python object, the loaded namespaces. The set of loaded /// namespaces is used during the import to verify if we can import a /// CLR assembly as a module or not. The set is stored on the clr module. /// - static void SetupNamespaceTracking () + static void SetupNamespaceTracking() { var newset = Runtime.PySet_New(new BorrowedReference(IntPtr.Zero)); try @@ -130,7 +181,7 @@ static void SetupNamespaceTracking () var pyNs = Runtime.PyString_FromString(ns); try { - if(Runtime.PySet_Add(newset, new BorrowedReference(pyNs)) != 0) + if (Runtime.PySet_Add(newset, new BorrowedReference(pyNs)) != 0) { throw new PythonException(); } @@ -141,7 +192,7 @@ static void SetupNamespaceTracking () } } - if(Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset.DangerousGetAddress()) != 0) + if (Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset.DangerousGetAddress()) != 0) { throw new PythonException(); } @@ -152,7 +203,7 @@ static void SetupNamespaceTracking () } AssemblyManager.namespaceAdded += OnNamespaceAdded; - PythonEngine.AddShutdownHandler(()=>AssemblyManager.namespaceAdded -= OnNamespaceAdded); + PythonEngine.AddShutdownHandler(() => AssemblyManager.namespaceAdded -= OnNamespaceAdded); } /// @@ -162,27 +213,21 @@ static void SetupNamespaceTracking () static void TeardownNameSpaceTracking() { AssemblyManager.namespaceAdded -= OnNamespaceAdded; - // If the C# runtime isn't loaded, then there is no namespaces available - if ((Runtime.PyDict_DelItemString(new BorrowedReference(root.dict), availableNsKey) != 0) && - (Exceptions.ExceptionMatches(Exceptions.KeyError))) - { - // Trying to remove a key that's not in the dictionary - // raises an error. We don't care about it. - Runtime.PyErr_Clear(); - } + // If the C# runtime isn't loaded, then there are no namespaces available + Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); } - static void OnNamespaceAdded (string name) + static void OnNamespaceAdded(string name) { - using(Py.GIL()) + using (Py.GIL()) { var pyNs = Runtime.PyString_FromString(name); try { var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); - if (!nsSet.IsNull) + if (!nsSet.IsNull || nsSet.DangerousGetAddress() != Runtime.PyNone) { - if(Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) + if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) { throw new PythonException(); } @@ -225,7 +270,7 @@ public static unsafe NewReference GetCLRModule() /// /// The hook to import a CLR module into Python /// - public static ModuleObject __import__(string modname) + public static ModuleObject Import(string modname) { // Traverse the qualified module name to get the named module. // Note that if diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 9eb1d997d..19178fe41 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -593,12 +593,12 @@ public static int _AtExit() /// A new reference to the imported module, as a PyObject. [ModuleFunction] [ForbidPythonThreads] - public static PyObject _LoadClrModule(PyObject spec) + public static PyObject _load_clr_module(PyObject spec) { ModuleObject mod = null; using (var modname = spec.GetAttr("name")) { - mod = ImportHook.__import__(modname.ToString()); + mod = ImportHook.Import(modname.ToString()); } // We can't return directly a ModuleObject, because the tpHandle is // not set, but we can return a PyObject. diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index d9c3ee52c..6e6da2d93 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -160,7 +160,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "getPreload", "Initialize", "ListAssemblies", - "_LoadClrModule", + "_load_clr_module", "Release", "Reset", "set_SuppressDocs", From 2af066db6da89e3cad1119b9d4408c24f533e6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 12 Apr 2021 15:46:59 -0400 Subject: [PATCH 134/151] fixup! Merge remote-tracking branch 'origin/master' into modernize-import-hook --- src/runtime/importhook.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index df1ae253e..4eaf3b97e 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -39,11 +39,11 @@ def __init__(self): @classmethod def find_spec(klass, fullname, paths=None, target=None): import clr - if clr._availableNamespaces and fullname in clr._availableNamespaces: + if clr._available_namespaces and fullname in clr._available_namespaces: return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) return None "; - const string availableNsKey = "_availableNamespaces"; + const string availableNsKey = "_available_namespaces"; /// /// Initialization performed on startup of the Python runtime. From bb490bf5f324f6f63f68bfb5cdb231f1d9934e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 12 Apr 2021 15:53:01 -0400 Subject: [PATCH 135/151] fixup! fixup! Merge remote-tracking branch 'origin/master' into modernize-import-hook --- src/runtime/runtime.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 789b71f3e..c94531c60 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1944,6 +1944,11 @@ internal static string PyModule_GetFilename(IntPtr module) internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); + internal static int PyModule_AddObject(IntPtr module, string name, IntPtr stolenObject) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_AddObject(module, namePtr, stolenObject); + } /// /// Return value: New reference. @@ -2502,6 +2507,7 @@ static Delegates() { PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); } + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2791,6 +2797,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } From 31ea87698811ac382243f7328bc0885bfb4e5c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 1 Jun 2021 11:21:49 -0400 Subject: [PATCH 136/151] Create a clr.loader module * Return ModuleObject.pyHandle, do not convert. * Write domain tests generated code to file. --- src/runtime/converter.cs | 8 +++ src/runtime/importhook.cs | 65 ++++++++---------------- src/runtime/moduleobject.cs | 15 ++---- src/runtime/runtime.cs | 6 +-- tests/domain_tests/TestRunner.cs | 13 ++++- tests/domain_tests/test_domain_reload.py | 1 - 6 files changed, 46 insertions(+), 62 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 47263e8c4..109c590c7 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -202,6 +202,14 @@ internal static IntPtr ToPython(object value, Type type) return ClassDerivedObject.ToPython(pyderived); } + // ModuleObjects are created in a way that their wrapping them as + // a CLRObject fails, the ClassObject has no tpHandle. Return the + // pyHandle as is, do not convert. + if (value is ModuleObject modobj) + { + return modobj.pyHandle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 4eaf3b97e..0e1076ce4 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Runtime.InteropServices; namespace Python.Runtime @@ -18,9 +19,6 @@ import sys class DotNetLoader(importlib.abc.Loader): - def __init__(self): - super().__init__() - @classmethod def exec_module(klass, mod): # This method needs to exist. @@ -32,13 +30,13 @@ import clr return clr._load_clr_module(spec) class DotNetFinder(importlib.abc.MetaPathFinder): - - def __init__(self): - super().__init__() - + @classmethod def find_spec(klass, fullname, paths=None, target=None): - import clr + # Don't import, we might call ourselves recursively! + if 'clr' not in sys.modules: + return None + clr = sys.modules['clr'] if clr._available_namespaces and fullname in clr._available_namespaces: return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) return None @@ -64,13 +62,10 @@ internal static unsafe void Initialize() BorrowedReference dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); - - // Add/create the MetaPathLoader SetupNamespaceTracking(); SetupImportHook(); } - /// /// Cleanup resources upon shutdown of the Python runtime. /// @@ -81,11 +76,10 @@ internal static void Shutdown() return; } - bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + TeardownNameSpaceTracking(); Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; - TeardownNameSpaceTracking(); Runtime.XDecref(root.pyHandle); root = null; CLRModule.Reset(); @@ -107,7 +101,6 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); BorrowedReference dict = Runtime.PyImport_GetModuleDict(); - Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "CLR", py_clr_module); Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "clr", py_clr_module); SetupNamespaceTracking(); } @@ -115,54 +108,35 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) static void SetupImportHook() { // Create the import hook module - var import_hook_module_def = ModuleDefOffset.AllocModuleDef("clr.loader"); - var import_hook_module = Runtime.PyModule_Create2(import_hook_module_def, 3); + var import_hook_module = Runtime.PyModule_New("clr.loader"); // Run the python code to create the module's classes. - var mod_dict = Runtime.PyModule_GetDict(new BorrowedReference(import_hook_module)); var builtins = Runtime.PyEval_GetBuiltins(); var exec = Runtime.PyDict_GetItemString(builtins, "exec"); using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); - var codeStr = Runtime.PyString_FromString(LoaderCode); + IntPtr codeStr = Runtime.PyString_FromString(LoaderCode); Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 0, codeStr); // PyTuple_SetItem steals a reference, mod_dict is borrowed. + var mod_dict = Runtime.PyModule_GetDict(import_hook_module); Runtime.XIncref(mod_dict.DangerousGetAddress()); Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 1, mod_dict.DangerousGetAddress()); Runtime.PyObject_Call(exec.DangerousGetAddress(), args.DangerousGetAddress(), IntPtr.Zero); - var loader = Runtime.PyDict_GetItemString(mod_dict, "DotNetLoader").DangerousGetAddressOrNull(); - Runtime.XIncref(loader); - - // Add the classes to the module - // PyModule_AddObject steals a reference only on success - if (Runtime.PyModule_AddObject(import_hook_module, "DotNetLoader", loader) != 0) - { - Runtime.XDecref(loader); - throw new PythonException(); - } - - var finder = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder").DangerousGetAddressOrNull(); - Runtime.XIncref(finder); - if (Runtime.PyModule_AddObject(import_hook_module, "DotNetFinder", finder) != 0) - { - Runtime.XDecref(finder); - throw new PythonException(); - } - // Set as a sub-module of clr. - Runtime.XIncref(import_hook_module); - if(Runtime.PyModule_AddObject(py_clr_module, "loader", import_hook_module) != 0) + if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module) != 0) { - Runtime.XDecref(import_hook_module); + Runtime.XDecref(import_hook_module.DangerousGetAddress()); throw new PythonException(); } // Finally, add the hook to the meta path - var finder_inst = Runtime.PyDict_GetItemString(mod_dict, "finder_inst").DangerousGetAddressOrNull(); - Runtime.XIncref(finder); + var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder"); + var finderCtorArgs = Runtime.PyTuple_New(0); + var finder_inst = Runtime.PyObject_CallObject(findercls.DangerousGetAddress(), finderCtorArgs); + Runtime.XDecref(finderCtorArgs); var metapath = Runtime.PySys_GetObject("meta_path"); - Runtime.PyList_Append(metapath, finder); + Runtime.PyList_Append(metapath, finder_inst); } /// @@ -268,7 +242,8 @@ public static unsafe NewReference GetCLRModule() } /// - /// The hook to import a CLR module into Python + /// The hook to import a CLR module into Python. Returns a new reference + /// to the module. /// public static ModuleObject Import(string modname) { @@ -305,7 +280,7 @@ public static ModuleObject Import(string modname) tail.LoadNames(); } } - + tail.IncrRefCount(); return tail; } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 19178fe41..28c8c2237 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -53,7 +53,7 @@ public ModuleObject(string name) var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); PythonException.ThrowIfIsNull(dictRef); dict = dictRef.DangerousMoveToPointer(); - + __all__ = Runtime.PyList_New(0); using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); @@ -576,13 +576,6 @@ public static string[] ListAssemblies(bool verbose) return names; } - [ModuleFunction] - public static int _AtExit() - { - return Runtime.AtExit(); - } - - /// /// Note: This should *not* be called directly. /// The function that get/import a CLR assembly as a python module. @@ -593,16 +586,14 @@ public static int _AtExit() /// A new reference to the imported module, as a PyObject. [ModuleFunction] [ForbidPythonThreads] - public static PyObject _load_clr_module(PyObject spec) + public static ModuleObject _load_clr_module(PyObject spec) { ModuleObject mod = null; using (var modname = spec.GetAttr("name")) { mod = ImportHook.Import(modname.ToString()); } - // We can't return directly a ModuleObject, because the tpHandle is - // not set, but we can return a PyObject. - return new PyObject(mod.pyHandle); + return mod; } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index c94531c60..1838d00e6 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1944,7 +1944,7 @@ internal static string PyModule_GetFilename(IntPtr module) internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); - internal static int PyModule_AddObject(IntPtr module, string name, IntPtr stolenObject) + internal static int PyModule_AddObject(BorrowedReference module, string name, BorrowedReference stolenObject) { using var namePtr = new StrPtr(name, Encoding.UTF8); return Delegates.PyModule_AddObject(module, namePtr, stolenObject); @@ -2507,7 +2507,7 @@ static Delegates() { PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); } - PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2797,7 +2797,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } - internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index 716fe079b..66fb4f894 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -30,6 +30,11 @@ namespace Python.DomainReloadTests /// which test case to run. That's because pytest assumes we'll run /// everything in one process, but we really want a clean process on each /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// ### Debugging tips: ### + /// * Running pytest with the `-s` argument prevents stdout capture by pytest + /// * Add a sleep into the python test case before the crash/failure, then while + /// sleeping, attach the debugger to the Python.TestDomainReload.exe process. /// /// class TestRunner @@ -1287,7 +1292,13 @@ static string CreateAssembly(string name, string code, bool exe = false) } parameters.ReferencedAssemblies.Add(netstandard); parameters.ReferencedAssemblies.Add(PythonDllLocation); - CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + // Write code to file so it can debugged. + var sourcePath = Path.Combine(TestPath, name+"_source.cs"); + using(var file = new StreamWriter(sourcePath)) + { + file.Write(code); + } + CompilerResults results = provider.CompileAssemblyFromFile(parameters, sourcePath); if (results.NativeCompilerReturnValue != 0) { var stderr = System.Console.Error; diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index 71b1e348e..e7a82ded2 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -89,6 +89,5 @@ def test_in_to_ref_param(): def test_nested_type(): _run_test("nested_type") -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_import_after_reload(): _run_test("import_after_reload") \ No newline at end of file From c02d5c626a400d4d0f2847d24ba958b0fc7af97a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Jun 2021 10:50:56 -0400 Subject: [PATCH 137/151] Test to test if the test passes --- src/runtime/BorrowedReference.cs | 1 + src/runtime/assemblymanager.cs | 24 ++++++++++++------------ src/runtime/importhook.cs | 19 +++++++++++++------ src/runtime/moduleobject.cs | 5 +++++ 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index bf8a91d3e..75070aac2 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -9,6 +9,7 @@ readonly ref struct BorrowedReference { readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; + public bool IsNone => this.pointer == Runtime.PyNone; /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index fdde2aeb1..5b4f465b1 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -39,7 +39,7 @@ internal class AssemblyManager internal static List pypath; // Triggered when a new namespace is added to the namespaces dictionary - public static event Action namespaceAdded; + // public static event Action namespaceAdded; private AssemblyManager() { @@ -287,17 +287,17 @@ internal static void ScanAssembly(Assembly assembly) if (ns != null) { namespaces[ns].TryAdd(assembly, string.Empty); - try - { - namespaceAdded?.Invoke(ns); - } - catch (Exception e) - { - // For some reason, exceptions happening here does... nothing. - // Even System.AccessViolationExceptions gets ignored. - Console.WriteLine($"Namespace added callback failed with: {e}"); - throw; - } + // try + // { + // namespaceAdded?.Invoke(ns); + // } + // catch (Exception e) + // { + // // For some reason, exceptions happening here does... nothing. + // // Even System.AccessViolationExceptions gets ignored. + // Console.WriteLine($"Namespace added callback failed with: {e}"); + // throw; + // } } if (ns != null && t.IsGenericTypeDefinition) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 0e1076ce4..8d99980cb 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -147,7 +147,7 @@ static void SetupImportHook() /// static void SetupNamespaceTracking() { - var newset = Runtime.PySet_New(new BorrowedReference(IntPtr.Zero)); + var newset = Runtime.PySet_New(default); try { foreach (var ns in AssemblyManager.GetNamespaces()) @@ -176,8 +176,8 @@ static void SetupNamespaceTracking() newset.Dispose(); } - AssemblyManager.namespaceAdded += OnNamespaceAdded; - PythonEngine.AddShutdownHandler(() => AssemblyManager.namespaceAdded -= OnNamespaceAdded); + // AssemblyManager.namespaceAdded += OnNamespaceAdded; + // PythonEngine.AddShutdownHandler(() => AssemblyManager.namespaceAdded -= OnNamespaceAdded); } /// @@ -186,20 +186,25 @@ static void SetupNamespaceTracking() /// static void TeardownNameSpaceTracking() { - AssemblyManager.namespaceAdded -= OnNamespaceAdded; + // AssemblyManager.namespaceAdded -= OnNamespaceAdded; // If the C# runtime isn't loaded, then there are no namespaces available Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); } - static void OnNamespaceAdded(string name) + public static void OnNamespaceAdded(string name) { + Console.WriteLine(System.Environment.StackTrace); + Console.WriteLine("OnNamespaceAdded: acquiring"); + Console.Out.Flush(); using (Py.GIL()) { + Console.WriteLine("OnNamespaceAdded: acquired"); + Console.Out.Flush(); var pyNs = Runtime.PyString_FromString(name); try { var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); - if (!nsSet.IsNull || nsSet.DangerousGetAddress() != Runtime.PyNone) + if (!(nsSet.IsNull && nsSet.IsNone)) { if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) { @@ -212,6 +217,8 @@ static void OnNamespaceAdded(string name) Runtime.XDecref(pyNs); } } + Console.WriteLine("OnNamespaceAdded: released"); + Console.Out.Flush(); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 28c8c2237..f0b6840b0 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -529,6 +529,11 @@ public static Assembly AddReference(string name) } // Classes that are not in a namespace needs an extra nudge to be found. ImportHook.UpdateCLRModuleDict(); + + // Heavyhanded but otherwise we'd need a "addedSinceLastCall". + foreach(var ns in AssemblyManager.GetNamespaces()){ + ImportHook.OnNamespaceAdded(ns); + } return assembly; } From 970a18904675dbc6677b0d4262c72290c9ea9896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Jun 2021 11:08:14 -0400 Subject: [PATCH 138/151] fix new exception usage --- src/runtime/importhook.cs | 10 +++++----- src/runtime/moduleobject.cs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 8d99980cb..69042185f 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -127,7 +127,7 @@ static void SetupImportHook() if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module) != 0) { Runtime.XDecref(import_hook_module.DangerousGetAddress()); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } // Finally, add the hook to the meta path @@ -157,7 +157,7 @@ static void SetupNamespaceTracking() { if (Runtime.PySet_Add(newset, new BorrowedReference(pyNs)) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } finally @@ -168,7 +168,7 @@ static void SetupNamespaceTracking() if (Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset.DangerousGetAddress()) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } finally @@ -208,7 +208,7 @@ public static void OnNamespaceAdded(string name) { if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -275,7 +275,7 @@ public static ModuleObject Import(string modname) if (!(mt is ModuleObject)) { Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } if (head == null) { diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index f0b6840b0..039562ef8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -196,7 +196,7 @@ public void LoadNames() { if (Runtime.PyList_Append(new BorrowedReference(__all__), pyname) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } finally From 059605bc249cbd7ca2e92e0aa642f8da6b3c3eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Jun 2021 13:59:33 -0400 Subject: [PATCH 139/151] kick the build because I can't repro From ff170e99b78ef411e54efd4d2d429799670d043f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Jun 2021 16:07:27 -0400 Subject: [PATCH 140/151] Add Namespaces to the import hook only through AddReference Don't piggyback on AssemblyManager's AssemblyLoadHandler method because it may be called from other threads, leading to deadlocks if it is called while Python code is executing --- src/runtime/assemblymanager.cs | 15 --------------- src/runtime/importhook.cs | 15 ++------------- src/runtime/moduleobject.cs | 10 +++++++--- 3 files changed, 9 insertions(+), 31 deletions(-) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 5b4f465b1..d44f5f666 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -37,10 +37,6 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; internal static List pypath; - - // Triggered when a new namespace is added to the namespaces dictionary - // public static event Action namespaceAdded; - private AssemblyManager() { } @@ -287,17 +283,6 @@ internal static void ScanAssembly(Assembly assembly) if (ns != null) { namespaces[ns].TryAdd(assembly, string.Empty); - // try - // { - // namespaceAdded?.Invoke(ns); - // } - // catch (Exception e) - // { - // // For some reason, exceptions happening here does... nothing. - // // Even System.AccessViolationExceptions gets ignored. - // Console.WriteLine($"Namespace added callback failed with: {e}"); - // throw; - // } } if (ns != null && t.IsGenericTypeDefinition) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 69042185f..acab01792 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -176,30 +176,21 @@ static void SetupNamespaceTracking() newset.Dispose(); } - // AssemblyManager.namespaceAdded += OnNamespaceAdded; - // PythonEngine.AddShutdownHandler(() => AssemblyManager.namespaceAdded -= OnNamespaceAdded); } /// - /// Removes the set of available namespaces from the clr module and - /// removes the callback on the OnNamespaceAdded event. + /// Removes the set of available namespaces from the clr module. /// static void TeardownNameSpaceTracking() { - // AssemblyManager.namespaceAdded -= OnNamespaceAdded; // If the C# runtime isn't loaded, then there are no namespaces available Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); } - public static void OnNamespaceAdded(string name) + public static void AddNamespace(string name) { - Console.WriteLine(System.Environment.StackTrace); - Console.WriteLine("OnNamespaceAdded: acquiring"); - Console.Out.Flush(); using (Py.GIL()) { - Console.WriteLine("OnNamespaceAdded: acquired"); - Console.Out.Flush(); var pyNs = Runtime.PyString_FromString(name); try { @@ -217,8 +208,6 @@ public static void OnNamespaceAdded(string name) Runtime.XDecref(pyNs); } } - Console.WriteLine("OnNamespaceAdded: released"); - Console.Out.Flush(); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 039562ef8..9dc09ba58 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -509,6 +509,7 @@ public static bool SuppressOverloads public static Assembly AddReference(string name) { AssemblyManager.UpdatePath(); + var origNs = AssemblyManager.GetNamespaces(); Assembly assembly = null; assembly = AssemblyManager.FindLoadedAssembly(name); if (assembly == null) @@ -530,9 +531,12 @@ public static Assembly AddReference(string name) // Classes that are not in a namespace needs an extra nudge to be found. ImportHook.UpdateCLRModuleDict(); - // Heavyhanded but otherwise we'd need a "addedSinceLastCall". - foreach(var ns in AssemblyManager.GetNamespaces()){ - ImportHook.OnNamespaceAdded(ns); + // A bit heavyhanded, but we can't use the AssemblyManager's AssemblyLoadHandler + // method because it may be called from other threads, leading to deadlocks + // if it is called while Python code is executing. + var currNs = AssemblyManager.GetNamespaces().Except(origNs); + foreach(var ns in currNs){ + ImportHook.AddNamespace(ns); } return assembly; } From da051a988de98d1fba23f3f35f5cf20f2f0258eb Mon Sep 17 00:00:00 2001 From: Peter Kese Date: Tue, 8 Jun 2021 14:35:46 +0200 Subject: [PATCH 141/151] Add a failing test for Unicode conversion --- src/embed_tests/TestPyString.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 0de436e35..561fd4eaa 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -94,5 +94,15 @@ public void TestUnicode() PyObject actual = new PyString(expected); Assert.AreEqual(expected, actual.ToString()); } + + [Test] + public void TestUnicodeSurrogate() + { + const string expected = "foo\ud83d\udc3c"; // "foo🐼" + PyObject actual = new PyString(expected); + // python treats "foo🐼" as 4 characters, dotnet as 5 + Assert.AreEqual(4, actual.Length()); + Assert.AreEqual(expected, actual.ToString()); + } } } From 1b6c6c059b5adc6b09d3673a1d4034c40902b777 Mon Sep 17 00:00:00 2001 From: Peter Kese Date: Tue, 8 Jun 2021 20:15:07 +0200 Subject: [PATCH 142/151] Fix Python -> .Net unicode string conversion --- src/runtime/runtime.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 789b71f3e..1065fda19 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1646,11 +1646,12 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); - int length = (int)PyUnicode_GetSize(op); - char* codePoints = (char*)PyBytes_AsString(p.DangerousGetAddress()); + var bytesPtr = p.DangerousGetAddress(); + int bytesLength = (int)Runtime.PyBytes_Size(bytesPtr); + char* codePoints = (char*)PyBytes_AsString(bytesPtr); return new string(codePoints, startIndex: 1, // skip BOM - length: length); + length: bytesLength/2-1); // utf16 - BOM } return null; From 4674b5c55d419a50801c6813ab4238db2d9860ad Mon Sep 17 00:00:00 2001 From: Peter Kese Date: Tue, 8 Jun 2021 20:44:33 +0200 Subject: [PATCH 143/151] Add separate test for the initial problem (passing); disable failing test --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/embed_tests/TestPyString.cs | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 6cfa216b1..ebd2021b3 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -55,6 +55,7 @@ - Meinrad Recheis ([@henon](https://github.com/henon)) - Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) +- Peter Kese ([@patstew](https://github.com/pkese)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5871e7ffb..c453f2b4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - Exception stacktraces on `PythonException.StackTrace` are now properly formatted - Providing an invalid type parameter to a generic type or method produces a helpful Python error - Empty parameter names (as can be generated from F#) do not cause crashes +- Unicode strings with surrogates get truncated when converting from Python ### Removed diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 561fd4eaa..c2f2a0d6a 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -96,6 +96,16 @@ public void TestUnicode() } [Test] + public void TestUnicodeSurrogateToString() + { + var expected = "foo\ud83d\udc3c"; + var actual = PythonEngine.Eval("'foo\ud83d\udc3c'"); + Assert.AreEqual(4, actual.Length()); + Assert.AreEqual(expected, actual.ToString()); + } + + [Test] + [Ignore("Bug: Unicode conversion issue #1466")] public void TestUnicodeSurrogate() { const string expected = "foo\ud83d\udc3c"; // "foo🐼" From 63de923a44259732525810bcbcca93bb963d0b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Jun 2021 15:16:28 -0400 Subject: [PATCH 144/151] Review changes, update API usage --- src/embed_tests/pyimport.cs | 2 +- src/runtime/BorrowedReference.cs | 1 - src/runtime/importhook.cs | 69 +++++++++++--------------------- src/runtime/moduleobject.cs | 8 ++-- src/runtime/pylist.cs | 2 +- src/runtime/runtime.cs | 26 ++++++++++-- 6 files changed, 51 insertions(+), 57 deletions(-) diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index e98461cbb..de8a06bf8 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -37,7 +37,7 @@ public void SetUp() Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Assert.IsFalse(path.IsNull); - Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.PyList_Append(path, new BorrowedReference(str)); Runtime.Runtime.XDecref(str); } diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 75070aac2..bf8a91d3e 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -9,7 +9,6 @@ readonly ref struct BorrowedReference { readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; - public bool IsNone => this.pointer == Runtime.PyNone; /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index acab01792..6ad9155a1 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -101,7 +101,7 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); BorrowedReference dict = Runtime.PyImport_GetModuleDict(); - Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "clr", py_clr_module); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); SetupNamespaceTracking(); } @@ -115,14 +115,12 @@ static void SetupImportHook() var exec = Runtime.PyDict_GetItemString(builtins, "exec"); using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); - IntPtr codeStr = Runtime.PyString_FromString(LoaderCode); - Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 0, codeStr); - // PyTuple_SetItem steals a reference, mod_dict is borrowed. + var codeStr = NewReference.DangerousFromPointer(Runtime.PyString_FromString(LoaderCode)); + Runtime.PyTuple_SetItem(args, 0, codeStr); var mod_dict = Runtime.PyModule_GetDict(import_hook_module); - Runtime.XIncref(mod_dict.DangerousGetAddress()); - Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 1, mod_dict.DangerousGetAddress()); - Runtime.PyObject_Call(exec.DangerousGetAddress(), args.DangerousGetAddress(), IntPtr.Zero); - + // reference not stolen due to overload incref'ing for us. + Runtime.PyTuple_SetItem(args, 1, mod_dict); + Runtime.PyObject_Call(exec, args, default); // Set as a sub-module of clr. if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module) != 0) { @@ -132,9 +130,8 @@ static void SetupImportHook() // Finally, add the hook to the meta path var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder"); - var finderCtorArgs = Runtime.PyTuple_New(0); - var finder_inst = Runtime.PyObject_CallObject(findercls.DangerousGetAddress(), finderCtorArgs); - Runtime.XDecref(finderCtorArgs); + var finderCtorArgs = NewReference.DangerousFromPointer(Runtime.PyTuple_New(0)); + var finder_inst = Runtime.PyObject_CallObject(findercls, finderCtorArgs); var metapath = Runtime.PySys_GetObject("meta_path"); Runtime.PyList_Append(metapath, finder_inst); } @@ -147,34 +144,19 @@ static void SetupImportHook() /// static void SetupNamespaceTracking() { - var newset = Runtime.PySet_New(default); - try + using var newset = Runtime.PySet_New(default); + foreach (var ns in AssemblyManager.GetNamespaces()) { - foreach (var ns in AssemblyManager.GetNamespaces()) + using var pyNs = NewReference.DangerousFromPointer(Runtime.PyString_FromString(ns)); + if (Runtime.PySet_Add(newset, pyNs) != 0) { - var pyNs = Runtime.PyString_FromString(ns); - try - { - if (Runtime.PySet_Add(newset, new BorrowedReference(pyNs)) != 0) - { - throw PythonException.ThrowLastAsClrException(); - } - } - finally - { - Runtime.XDecref(pyNs); - } + throw PythonException.ThrowLastAsClrException(); } - - if (Runtime.PyDict_SetItemString(root.dict, availableNsKey, newset.DangerousGetAddress()) != 0) + if (Runtime.PyDict_SetItemString(root.DictRef, availableNsKey, newset) != 0) { throw PythonException.ThrowLastAsClrException(); } } - finally - { - newset.Dispose(); - } } @@ -189,24 +171,21 @@ static void TeardownNameSpaceTracking() public static void AddNamespace(string name) { - using (Py.GIL()) + var pyNs = Runtime.PyString_FromString(name); + try { - var pyNs = Runtime.PyString_FromString(name); - try + var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); + if (!(nsSet.IsNull || nsSet.DangerousGetAddress() == Runtime.PyNone)) { - var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); - if (!(nsSet.IsNull && nsSet.IsNone)) + if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) { - if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) - { - throw PythonException.ThrowLastAsClrException(); - } + throw PythonException.ThrowLastAsClrException(); } } - finally - { - Runtime.XDecref(pyNs); - } + } + finally + { + Runtime.XDecref(pyNs); } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 9dc09ba58..c2614b1d8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -194,7 +194,7 @@ public void LoadNames() var pyname = Runtime.PyString_FromString(name); try { - if (Runtime.PyList_Append(new BorrowedReference(__all__), pyname) != 0) + if (Runtime.PyList_Append(new BorrowedReference(__all__), new BorrowedReference(pyname)) != 0) { throw PythonException.ThrowLastAsClrException(); } @@ -598,10 +598,8 @@ public static string[] ListAssemblies(bool verbose) public static ModuleObject _load_clr_module(PyObject spec) { ModuleObject mod = null; - using (var modname = spec.GetAttr("name")) - { - mod = ImportHook.Import(modname.ToString()); - } + using var modname = spec.GetAttr("name"); + mod = ImportHook.Import(modname.ToString()); return mod; } } diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 039f5e313..8f346524f 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -132,7 +132,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(this.Reference, item.obj); + int r = Runtime.PyList_Append(this.Reference, new BorrowedReference(item.obj)); if (r < 0) { throw PythonException.ThrowLastAsClrException(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1838d00e6..b3979c5c9 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -174,7 +174,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd IntPtr item = PyString_FromString(rtdir); if (PySequence_Contains(path, item) == 0) { - PyList_Append(new BorrowedReference(path), item); + PyList_Append(new BorrowedReference(path), new BorrowedReference(item)); } XDecref(item); AssemblyManager.UpdatePath(); @@ -1087,6 +1087,8 @@ internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); + internal static IntPtr PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) + => Delegates.PyObject_Call(pointer.DangerousGetAddress(), args.DangerousGetAddress(), kw.DangerousGetAddressOrNull()); internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); @@ -1822,7 +1824,7 @@ internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); - internal static int PyList_Append(BorrowedReference pointer, IntPtr value) => Delegates.PyList_Append(pointer, value); + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); @@ -1885,7 +1887,15 @@ internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) { return PyTuple_SetItem(pointer, new IntPtr(index), value); } + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, StolenReference value) + => PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), value.DangerousGetAddressOrNull()); + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, BorrowedReference value) + { + var increfValue = value.DangerousGetAddress(); + Runtime.XIncref(increfValue); + return PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), increfValue); + } private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); @@ -1944,6 +1954,14 @@ internal static string PyModule_GetFilename(IntPtr module) internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); + + /// + /// We can't use a StolenReference here because the reference is stolen only on success. + /// + /// The module to add the object to. + /// The key that will refer to the object. + /// The object to add to the module. + /// Return -1 on error, 0 on success. internal static int PyModule_AddObject(BorrowedReference module, string name, BorrowedReference stolenObject) { using var namePtr = new StrPtr(name, Encoding.UTF8); @@ -2483,7 +2501,7 @@ static Delegates() PyList_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetItem), GetUnmanagedDll(_PythonDll)); PyList_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetItem), GetUnmanagedDll(_PythonDll)); PyList_Insert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Insert), GetUnmanagedDll(_PythonDll)); - PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); + PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); PyList_Reverse = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Reverse), GetUnmanagedDll(_PythonDll)); PyList_Sort = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Sort), GetUnmanagedDll(_PythonDll)); PyList_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetSlice), GetUnmanagedDll(_PythonDll)); @@ -2780,7 +2798,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyList_GetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_SetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_Insert { get; } - internal static delegate* unmanaged[Cdecl] PyList_Append { get; } + internal static delegate* unmanaged[Cdecl] PyList_Append { get; } internal static delegate* unmanaged[Cdecl] PyList_Reverse { get; } internal static delegate* unmanaged[Cdecl] PyList_Sort { get; } internal static delegate* unmanaged[Cdecl] PyList_GetSlice { get; } From 1629116c08ac90759b1c4283b156078c394bd710 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 9 Jun 2021 08:09:34 +0200 Subject: [PATCH 145/151] Use exclusively PyUnicode_DecodeUTF16 for .NET->Python string conversions --- src/embed_tests/TestCustomMarshal.cs | 2 +- src/embed_tests/TestRuntime.cs | 2 +- src/runtime/converter.cs | 2 +- src/runtime/exceptions.cs | 6 ++-- src/runtime/pystring.cs | 2 +- src/runtime/runtime.cs | 47 ++++++---------------------- src/runtime/typemanager.cs | 2 +- 7 files changed, 17 insertions(+), 46 deletions(-) diff --git a/src/embed_tests/TestCustomMarshal.cs b/src/embed_tests/TestCustomMarshal.cs index 5860857a3..99911bdb0 100644 --- a/src/embed_tests/TestCustomMarshal.cs +++ b/src/embed_tests/TestCustomMarshal.cs @@ -23,7 +23,7 @@ public static void GetManagedStringTwice() { const string expected = "FooBar"; - IntPtr op = Runtime.Runtime.PyUnicode_FromString(expected); + IntPtr op = Runtime.Runtime.PyString_FromString(expected); string s1 = Runtime.Runtime.GetManagedString(op); string s2 = Runtime.Runtime.GetManagedString(op); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 9ca6cf139..9fb2e8b22 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -36,7 +36,7 @@ public static void Py_IsInitializedValue() public static void RefCountTest() { Runtime.Runtime.Py_Initialize(); - IntPtr op = Runtime.Runtime.PyUnicode_FromString("FooBar"); + IntPtr op = Runtime.Runtime.PyString_FromString("FooBar"); // New object RefCount should be one Assert.AreEqual(1, Runtime.Runtime.Refcount(op)); diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 47263e8c4..80f31f058 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -221,7 +221,7 @@ internal static IntPtr ToPython(object value, Type type) return CLRObject.GetInstHandle(value, type); case TypeCode.String: - return Runtime.PyUnicode_FromString((string)value); + return Runtime.PyString_FromString((string)value); case TypeCode.Int32: return Runtime.PyInt_FromInt32((int)value); diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index bbdcdad30..a612e34e3 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -50,7 +50,7 @@ internal static Exception ToException(BorrowedReference ob) { message = String.Format("{0}()", name); } - return Runtime.PyUnicode_FromString(message); + return Runtime.PyString_FromString(message); } /// @@ -75,7 +75,7 @@ internal static Exception ToException(BorrowedReference ob) { message = message.Substring(fullTypeName.Length); } - return Runtime.PyUnicode_FromString(message); + return Runtime.PyString_FromString(message); } } @@ -153,7 +153,7 @@ internal static void SetArgsAndCause(BorrowedReference ob, Exception e) if (!string.IsNullOrEmpty(e.Message)) { args = Runtime.PyTuple_New(1); - IntPtr msg = Runtime.PyUnicode_FromString(e.Message); + IntPtr msg = Runtime.PyString_FromString(e.Message); Runtime.PyTuple_SetItem(args, 0, msg); } else diff --git a/src/runtime/pystring.cs b/src/runtime/pystring.cs index 07eabba14..172c09ebd 100644 --- a/src/runtime/pystring.cs +++ b/src/runtime/pystring.cs @@ -51,7 +51,7 @@ public PyString(PyObject o) : base(FromObject(o)) private static IntPtr FromString(string s) { - IntPtr val = Runtime.PyUnicode_FromUnicode(s, s.Length); + IntPtr val = Runtime.PyString_FromString(s); PythonException.ThrowIfIsNull(val); return val; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1065fda19..1d736f1ec 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -230,7 +230,7 @@ private static void InitPyMembers() () => PyStringType = IntPtr.Zero); XDecref(op); - op = PyUnicode_FromString("unicode"); + op = PyString_FromString("unicode"); SetPyMemberTypeOf(ref PyUnicodeType, op, () => PyUnicodeType = IntPtr.Zero); XDecref(op); @@ -1527,7 +1527,12 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { fixed(char* ptr = value) - return PyUnicode_FromKindAndData(2, (IntPtr)ptr, value.Length); + return Delegates.PyUnicode_DecodeUTF16( + (IntPtr)ptr, + new IntPtr(value.Length * sizeof(Char)), + IntPtr.Zero, + IntPtr.Zero + ).DangerousGetAddress(); } @@ -1553,16 +1558,6 @@ internal static long PyBytes_Size(IntPtr op) private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); - - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) - { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); - } - - - private static IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size) => Delegates.PyUnicode_FromStringAndSize(value, size); - - internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode); internal static bool PyUnicode_Check(IntPtr ob) @@ -1576,22 +1571,6 @@ internal static bool PyUnicode_Check(IntPtr ob) internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - - - private static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, IntPtr size) - => Delegates.PyUnicode_FromKindAndData(kind, s, size); - - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - fixed(char* ptr = s) - return PyUnicode_FromKindAndData(2, (IntPtr)ptr, size); - } - - internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax(); internal static long PyUnicode_GetSize(IntPtr ob) @@ -1610,12 +1589,6 @@ internal static long PyUnicode_GetSize(IntPtr ob) internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); - internal static IntPtr PyUnicode_FromString(string s) - { - return PyUnicode_FromUnicode(s, s.Length); - } - - internal static IntPtr PyUnicode_InternFromString(string s) { using var ptr = new StrPtr(s, Encoding.UTF8); @@ -2443,11 +2416,10 @@ static Delegates() PyBytes_AsString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll)); PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); - PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); + PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); - PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll)); PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); @@ -2739,11 +2711,10 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyBytes_AsString { get; } internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_FromStringAndSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_FromKindAndData { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 13d822c09..e1bfe6aef 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -580,7 +580,7 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. - IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); From df5ebc2ec4eb55411fbfb2113a6305ec927d6044 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 9 Jun 2021 08:35:34 +0200 Subject: [PATCH 146/151] Activate UnicodeSurrogate test --- src/embed_tests/TestPyString.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index c2f2a0d6a..669ecde0d 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -105,7 +105,6 @@ public void TestUnicodeSurrogateToString() } [Test] - [Ignore("Bug: Unicode conversion issue #1466")] public void TestUnicodeSurrogate() { const string expected = "foo\ud83d\udc3c"; // "foo🐼" From d084b2e0d53293d26bb2a72d7e078306fa9cdaf2 Mon Sep 17 00:00:00 2001 From: Peter Kese Date: Wed, 9 Jun 2021 18:40:35 +0200 Subject: [PATCH 147/151] Apply suggested changes --- CHANGELOG.md | 2 +- src/runtime/runtime.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c453f2b4b..8aba9e9b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - Exception stacktraces on `PythonException.StackTrace` are now properly formatted - Providing an invalid type parameter to a generic type or method produces a helpful Python error - Empty parameter names (as can be generated from F#) do not cause crashes -- Unicode strings with surrogates get truncated when converting from Python +- Unicode strings with surrogates were truncated when converting from Python ### Removed diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1d736f1ec..80a0e5f01 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1619,12 +1619,12 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); - var bytesPtr = p.DangerousGetAddress(); - int bytesLength = (int)Runtime.PyBytes_Size(bytesPtr); + var bytesPtr = p.DangerousMoveToPointerOrNull(); + nint bytesLength = (nint)Runtime.PyBytes_Size(bytesPtr); char* codePoints = (char*)PyBytes_AsString(bytesPtr); return new string(codePoints, startIndex: 1, // skip BOM - length: bytesLength/2-1); // utf16 - BOM + length: (int) (bytesLength/2-1)); // utf16 - BOM } return null; From f061d287e43407d96a8ac4c75b8a7a030f45a80e Mon Sep 17 00:00:00 2001 From: Peter Kese Date: Wed, 9 Jun 2021 18:58:02 +0200 Subject: [PATCH 148/151] Revert stuff that I don't understand --- AUTHORS.md | 2 +- src/runtime/runtime.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index ebd2021b3..912831836 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -55,7 +55,7 @@ - Meinrad Recheis ([@henon](https://github.com/henon)) - Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) -- Peter Kese ([@patstew](https://github.com/pkese)) +- Peter Kese ([@pkese](https://github.com/pkese)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 80a0e5f01..1d736f1ec 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1619,12 +1619,12 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); - var bytesPtr = p.DangerousMoveToPointerOrNull(); - nint bytesLength = (nint)Runtime.PyBytes_Size(bytesPtr); + var bytesPtr = p.DangerousGetAddress(); + int bytesLength = (int)Runtime.PyBytes_Size(bytesPtr); char* codePoints = (char*)PyBytes_AsString(bytesPtr); return new string(codePoints, startIndex: 1, // skip BOM - length: (int) (bytesLength/2-1)); // utf16 - BOM + length: bytesLength/2-1); // utf16 - BOM } return null; From 49ccc1e5cf4d1af3cca1ee937bc3db69299cd2da Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 11 Jun 2021 07:27:29 +0200 Subject: [PATCH 149/151] Apply code review suggestions --- src/runtime/runtime.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1d736f1ec..537e5348f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1529,10 +1529,10 @@ internal static IntPtr PyString_FromString(string value) fixed(char* ptr = value) return Delegates.PyUnicode_DecodeUTF16( (IntPtr)ptr, - new IntPtr(value.Length * sizeof(Char)), + value.Length * sizeof(Char), IntPtr.Zero, IntPtr.Zero - ).DangerousGetAddress(); + ).DangerousMoveToPointerOrNull(); } @@ -2418,7 +2418,7 @@ static Delegates() _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); - PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); + PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); @@ -2714,7 +2714,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } From bd7e7450847dec33f9906d95270eae61fb8eead6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 14 Jun 2021 09:57:53 -0400 Subject: [PATCH 150/151] make PyModule_AddObject in line with CPython --- src/runtime/importhook.cs | 2 +- src/runtime/runtime.cs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 6ad9155a1..1111adc28 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -122,7 +122,7 @@ static void SetupImportHook() Runtime.PyTuple_SetItem(args, 1, mod_dict); Runtime.PyObject_Call(exec, args, default); // Set as a sub-module of clr. - if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module) != 0) + if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module.DangerousGetAddress()) != 0) { Runtime.XDecref(import_hook_module.DangerousGetAddress()); throw PythonException.ThrowLastAsClrException(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec3d64000..009412ea5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1934,9 +1934,12 @@ internal static string PyModule_GetFilename(IntPtr module) /// /// The module to add the object to. /// The key that will refer to the object. - /// The object to add to the module. + /// + /// The object to add to the module. The reference will be stolen only if the + /// method returns 0. + /// /// Return -1 on error, 0 on success. - internal static int PyModule_AddObject(BorrowedReference module, string name, BorrowedReference stolenObject) + internal static int PyModule_AddObject(BorrowedReference module, string name, IntPtr stolenObject) { using var namePtr = new StrPtr(name, Encoding.UTF8); return Delegates.PyModule_AddObject(module, namePtr, stolenObject); @@ -2498,7 +2501,7 @@ static Delegates() { PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); } - PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2787,7 +2790,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } - internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } From 46a85fedd1197a3ff3d4f2df337e5d900639b696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 15 Jun 2021 09:33:45 -0400 Subject: [PATCH 151/151] take care of stragglers --- src/runtime/converter.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index e19a62618..7cee0890c 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -207,7 +207,9 @@ internal static IntPtr ToPython(object value, Type type) // pyHandle as is, do not convert. if (value is ModuleObject modobj) { - return modobj.pyHandle; + var handle = modobj.pyHandle; + Runtime.XIncref(handle); + return handle; } // hmm - from Python, we almost never care what the declared