From 4a84a1e35f072fe506e1f9d0d85b45ecdf0bbcdf Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 14:42:46 -0800 Subject: [PATCH 1/9] NewReference type and an example usage --- src/runtime/NewReference.cs | 26 ++++++++++++++++++++++++++ src/runtime/NonCopyableAttribute.cs | 6 ++++++ src/runtime/Python.Runtime.15.csproj | 9 ++++++++- src/runtime/pydict.cs | 7 ++++--- src/runtime/runtime.cs | 2 +- 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 src/runtime/NewReference.cs create mode 100644 src/runtime/NonCopyableAttribute.cs diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs new file mode 100644 index 000000000..d9f6f8a37 --- /dev/null +++ b/src/runtime/NewReference.cs @@ -0,0 +1,26 @@ +namespace Python.Runtime +{ + using System; + [NonCopyable] + ref struct NewReference + { + public IntPtr Pointer { get; set; } + public bool IsNull => this.Pointer == IntPtr.Zero; + + public PyObject ToPyObject() + { + if (this.IsNull) throw new NullReferenceException(); + + var result = new PyObject(this.Pointer); + this.Pointer = IntPtr.Zero; + return result; + } + + public void Dispose() + { + if (!this.IsNull) + Runtime.XDecref(this.Pointer); + this.Pointer = IntPtr.Zero; + } + } +} diff --git a/src/runtime/NonCopyableAttribute.cs b/src/runtime/NonCopyableAttribute.cs new file mode 100644 index 000000000..63d36ab42 --- /dev/null +++ b/src/runtime/NonCopyableAttribute.cs @@ -0,0 +1,6 @@ +namespace Python.Runtime +{ + using System; + [AttributeUsage(AttributeTargets.Struct)] + class NonCopyableAttribute : Attribute { } +} diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index c31d4bf91..b4570499b 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -30,7 +30,7 @@ ..\..\ $(SolutionDir)\bin\ $(PythonBuildDir)\$(TargetFramework)\ - 7.3 + 8.0 True ..\pythonnet.snk $(PYTHONNET_DEFINE_CONSTANTS) @@ -129,6 +129,13 @@ + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 7237d1990..53582ad26 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -139,12 +139,13 @@ public PyObject Values() /// public PyObject Items() { - IntPtr items = Runtime.PyDict_Items(obj); - if (items == IntPtr.Zero) + using var items = Runtime.PyDict_Items(this.obj); + if (items.IsNull) { throw new PythonException(); } - return new PyObject(items); + + return items.ToPyObject(); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7748bafa9..d550fd83f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1591,7 +1591,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static extern IntPtr PyDict_Values(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Items(IntPtr pointer); + internal static extern NewReference PyDict_Items(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Copy(IntPtr pointer); From 88944a2136e42e6cc4d38da3889d3c6ea2bffcd2 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:12:30 -0800 Subject: [PATCH 2/9] BorrowedReference + example, that exposes dangerous pattern --- src/runtime/BorrowedReference.cs | 26 ++++++++++++++++++++++++++ src/runtime/assemblymanager.cs | 4 ++-- src/runtime/methodbinder.cs | 2 +- src/runtime/runtime.cs | 6 ++++-- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 src/runtime/BorrowedReference.cs diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs new file mode 100644 index 000000000..17f0960fe --- /dev/null +++ b/src/runtime/BorrowedReference.cs @@ -0,0 +1,26 @@ +namespace Python.Runtime +{ + using System; + [NonCopyable] + ref struct BorrowedReference + { + public IntPtr Pointer; + public bool IsNull => this.Pointer == IntPtr.Zero; + + public PyObject ToPyObject() + { + if (this.IsNull) throw new NullReferenceException(); + + Runtime.XIncref(this.Pointer); + return new PyObject(this.Pointer); + } + } + + static class BorrowedReferenceExtensions { + [Obsolete("Use overloads, that take BorrowedReference or NewReference")] + public static IntPtr DangerousGetAddress(this in BorrowedReference reference) + => reference.IsNull() ? throw new NullReferenceException() : reference.Pointer; + public static bool IsNull(this in BorrowedReference reference) + => reference.Pointer == IntPtr.Zero; + } +} diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..9d0296d47 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -145,7 +145,7 @@ internal static void UpdatePath() probed.Clear(); for (var i = 0; i < count; i++) { - IntPtr item = Runtime.PyList_GetItem(list, i); + BorrowedReference item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { @@ -492,4 +492,4 @@ internal static Type[] GetTypes(Assembly a) } } } -} \ No newline at end of file +} diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 8a7fc1930..4e8698da1 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -292,7 +292,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth for (int i = 0; i < pynkwargs; ++i) { var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(keylist, i)); - kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i); + kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i).DangerousGetAddress(); } Runtime.XDecref(keylist); Runtime.XDecref(valueList); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d550fd83f..4cceea42d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1509,6 +1509,8 @@ internal static IntPtr PyUnicode_FromString(string s) return PyUnicode_FromUnicode(s, s.Length); } + internal static string GetManagedString(in BorrowedReference borrowedReference) + => GetManagedString(borrowedReference.DangerousGetAddress()); ///

/// Function to access the internal PyUnicode/PyString object and /// convert it to a managed string with the correct encoding. @@ -1631,13 +1633,13 @@ internal static IntPtr PyList_New(long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) + internal static BorrowedReference PyList_GetItem(IntPtr pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); + private static extern BorrowedReference PyList_GetItem(IntPtr pointer, IntPtr index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) { From e1ba92c4eb0b9194e825799f4596add5d63077e0 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:25:03 -0800 Subject: [PATCH 3/9] actually, borrowed refrences can be copied --- src/runtime/BorrowedReference.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 17f0960fe..5c2a9fc2c 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -1,7 +1,6 @@ namespace Python.Runtime { using System; - [NonCopyable] ref struct BorrowedReference { public IntPtr Pointer; From 2d11f829c3fcbfafd043ce3f1ca1518445f4ff9b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:37:15 -0800 Subject: [PATCH 4/9] make BorrowedReference readonly ref struct --- src/runtime/BorrowedReference.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 5c2a9fc2c..00644ebe5 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -1,9 +1,9 @@ namespace Python.Runtime { using System; - ref struct BorrowedReference + readonly ref struct BorrowedReference { - public IntPtr Pointer; + public readonly IntPtr Pointer; public bool IsNull => this.Pointer == IntPtr.Zero; public PyObject ToPyObject() @@ -13,13 +13,14 @@ public PyObject ToPyObject() Runtime.XIncref(this.Pointer); return new PyObject(this.Pointer); } - } - static class BorrowedReferenceExtensions { [Obsolete("Use overloads, that take BorrowedReference or NewReference")] - public static IntPtr DangerousGetAddress(this in BorrowedReference reference) - => reference.IsNull() ? throw new NullReferenceException() : reference.Pointer; - public static bool IsNull(this in BorrowedReference reference) - => reference.Pointer == IntPtr.Zero; + public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.Pointer; + + BorrowedReference(IntPtr pointer) + { + this.Pointer = pointer; + } } } From dbb3dc539884146038c7567f2071480abf00b54a Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:43:13 -0800 Subject: [PATCH 5/9] updated old project file --- src/runtime/Python.Runtime.csproj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 0c2f912de..8cf39d6ff 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -15,7 +15,7 @@ ..\..\ $(SolutionDir)\bin\ Properties - 7.3 + 8.0 true false ..\pythonnet.snk @@ -83,6 +83,7 @@ + @@ -119,6 +120,8 @@ + + From 7304b25cf336264c8c078110d5418fbc2598a475 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:51:18 -0800 Subject: [PATCH 6/9] BorrowedReference.Pointer is a private readonly field --- src/runtime/BorrowedReference.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 00644ebe5..9b533e9d0 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -3,24 +3,24 @@ namespace Python.Runtime using System; readonly ref struct BorrowedReference { - public readonly IntPtr Pointer; - public bool IsNull => this.Pointer == IntPtr.Zero; + readonly IntPtr pointer; + public bool IsNull => this.pointer == IntPtr.Zero; public PyObject ToPyObject() { if (this.IsNull) throw new NullReferenceException(); - Runtime.XIncref(this.Pointer); - return new PyObject(this.Pointer); + Runtime.XIncref(this.pointer); + return new PyObject(this.pointer); } [Obsolete("Use overloads, that take BorrowedReference or NewReference")] public IntPtr DangerousGetAddress() - => this.IsNull ? throw new NullReferenceException() : this.Pointer; + => this.IsNull ? throw new NullReferenceException() : this.pointer; BorrowedReference(IntPtr pointer) { - this.Pointer = pointer; + this.pointer = pointer; } } } From 2bc218c23e2093b068c747c0a9e9890f441b4127 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 13 Feb 2020 15:58:35 -0800 Subject: [PATCH 7/9] drop C# 8.0 requirement --- src/runtime/Python.Runtime.15.csproj | 2 +- src/runtime/Python.Runtime.csproj | 2 +- src/runtime/pydict.cs | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index b4570499b..fd4f3416a 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -30,7 +30,7 @@ ..\..\ $(SolutionDir)\bin\ $(PythonBuildDir)\$(TargetFramework)\ - 8.0 + 7.3 True ..\pythonnet.snk $(PYTHONNET_DEFINE_CONSTANTS) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 8cf39d6ff..c79afee3e 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -15,7 +15,7 @@ ..\..\ $(SolutionDir)\bin\ Properties - 8.0 + 7.3 true false ..\pythonnet.snk diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 53582ad26..6d61cff76 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -139,13 +139,20 @@ public PyObject Values() /// public PyObject Items() { - using var items = Runtime.PyDict_Items(this.obj); - if (items.IsNull) + var items = Runtime.PyDict_Items(this.obj); + try { - throw new PythonException(); - } + if (items.IsNull) + { + throw new PythonException(); + } - return items.ToPyObject(); + return items.ToPyObject(); + } + finally + { + items.Dispose(); + } } From 2cf9fc36872789bbbb863a173addb4b35b7c5e95 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 23 Feb 2020 11:12:58 -0800 Subject: [PATCH 8/9] renamed NewReference.ToPyObject to MoveToPyObject removed public property Pointer from NewReference and replaced with DangerousGetAddress --- src/runtime/NewReference.cs | 18 +++++++++++------- src/runtime/pydict.cs | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index d9f6f8a37..f66b3bca3 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -4,23 +4,27 @@ namespace Python.Runtime [NonCopyable] ref struct NewReference { - public IntPtr Pointer { get; set; } - public bool IsNull => this.Pointer == IntPtr.Zero; + IntPtr pointer; + public bool IsNull => this.pointer == IntPtr.Zero; - public PyObject ToPyObject() + ///

Gets a raw pointer to the Python object

+ public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.pointer; + + public PyObject MoveToPyObject() { if (this.IsNull) throw new NullReferenceException(); - var result = new PyObject(this.Pointer); - this.Pointer = IntPtr.Zero; + var result = new PyObject(this.pointer); + this.pointer = IntPtr.Zero; return result; } public void Dispose() { if (!this.IsNull) - Runtime.XDecref(this.Pointer); - this.Pointer = IntPtr.Zero; + Runtime.XDecref(this.pointer); + this.pointer = IntPtr.Zero; } } } diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 6d61cff76..b396f4f3d 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -147,7 +147,7 @@ public PyObject Items() throw new PythonException(); } - return items.ToPyObject(); + return items.MoveToPyObject(); } finally { From 6ae63cd9678cd42545cd30b9be336b8ec5240045 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 23 Feb 2020 11:13:24 -0800 Subject: [PATCH 9/9] added xmldoc comments to *Reference members --- src/runtime/BorrowedReference.cs | 14 +++++--------- src/runtime/NewReference.cs | 11 ++++++++++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 9b533e9d0..7dbc7a811 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -1,20 +1,16 @@ namespace Python.Runtime { using System; + ///

+ /// Represents a reference to a Python object, that is being lent, and + /// can only be safely used until execution returns to the caller. + ///

readonly ref struct BorrowedReference { readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; - public PyObject ToPyObject() - { - if (this.IsNull) throw new NullReferenceException(); - - Runtime.XIncref(this.pointer); - return new PyObject(this.pointer); - } - - [Obsolete("Use overloads, that take BorrowedReference or NewReference")] + ///

Gets a raw pointer to the Python object

public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index f66b3bca3..3b45f821f 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -1,6 +1,9 @@ namespace Python.Runtime { using System; + ///

+ /// Represents a reference to a Python object, that is tracked by Python's reference counting. + ///

[NonCopyable] ref struct NewReference { @@ -11,6 +14,10 @@ ref struct NewReference public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; + ///

+ /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + ///

public PyObject MoveToPyObject() { if (this.IsNull) throw new NullReferenceException(); @@ -19,7 +26,9 @@ public PyObject MoveToPyObject() this.pointer = IntPtr.Zero; return result; } - + ///

+ /// Removes this reference to a Python object, and sets it to null. + ///

public void Dispose() { if (!this.IsNull)