From 28143d5f348a0edd891a2d8fa5249f7aab98582f Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Wed, 11 Sep 2019 21:59:27 -0700 Subject: [PATCH 01/11] enable expanding set of marshaling conversions via PyObjectConversions added sample TupleCodec (only supporting ValueTuple) --- src/embed_tests/Codecs.cs | 86 ++++++++ .../Python.EmbeddingTest.15.csproj | 3 +- src/embed_tests/Python.EmbeddingTest.csproj | 3 +- src/embed_tests/packages.config | 5 +- src/runtime/Codecs/TupleCodecs.cs | 129 ++++++++++++ src/runtime/Python.Runtime.csproj | 6 +- src/runtime/converter.cs | 24 ++- src/runtime/converterextensions.cs | 185 ++++++++++++++++++ src/runtime/pythonengine.cs | 2 + src/runtime/runtime.cs | 9 + 10 files changed, 444 insertions(+), 8 deletions(-) create mode 100644 src/embed_tests/Codecs.cs create mode 100644 src/runtime/Codecs/TupleCodecs.cs create mode 100644 src/runtime/converterextensions.cs diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..600215cf0 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,86 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 4f6b2de46..c335135b7 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 6 + 7.3 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT @@ -81,6 +81,7 @@ + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index faa55fa27..d8488011a 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -14,7 +14,7 @@ 1591 ..\..\ $(SolutionDir)\bin\ - 6 + 7.3 true prompt @@ -80,6 +80,7 @@ + diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..4052311fb 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - + - + + \ No newline at end of file diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs new file mode 100644 index 000000000..7c01eee46 --- /dev/null +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -0,0 +1,129 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder + { + TupleCodec() { } + public static TupleCodec Instance { get; } = new TupleCodec(); + + public bool CanEncode(Type type) + => type.Namespace == typeof(TTuple).Namespace && type.Name.StartsWith(typeof(TTuple).Name + '`') + || type == typeof(object) || type == typeof(TTuple); + + public PyObject TryEncode(object value) + { + if (value == null) return null; + + var tupleType = value.GetType(); + if (tupleType == typeof(object)) return null; + if (!this.CanEncode(tupleType)) return null; + if (tupleType == typeof(TTuple)) return new PyTuple(); + + long fieldCount = tupleType.GetGenericArguments().Length; + var tuple = Runtime.PyTuple_New(fieldCount); + Exceptions.ErrorCheck(tuple); + int fieldIndex = 0; + foreach (FieldInfo field in tupleType.GetFields()) + { + var item = field.GetValue(value); + IntPtr pyItem = Converter.ToPython(item); + Runtime.XIncref(pyItem); + Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); + fieldIndex++; + } + return new PyTuple(Runtime.SelfIncRef(tuple)); + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == Runtime.PyTupleType && this.CanEncode(targetType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + value = default; + + if (!Runtime.PyTuple_Check(pyObj.Handle)) return false; + + if (typeof(T) == typeof(object)) + { + bool converted = Decode(pyObj, out object result); + if (converted) + { + value = (T)result; + return true; + } + + return false; + } + + var itemTypes = typeof(T).GetGenericArguments(); + long itemCount = Runtime.PyTuple_Size(pyObj.Handle); + if (itemTypes.Length != itemCount) return false; + + if (itemCount == 0) + { + value = (T)EmptyTuple; + return true; + } + + var elements = new object[itemCount]; + for (int itemIndex = 0; itemIndex < itemTypes.Length; itemIndex++) + { + IntPtr pyItem = Runtime.PyTuple_GetItem(pyObj.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, itemTypes[itemIndex], out elements[itemIndex], setError: false)) + { + return false; + } + } + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = (T)factory.Invoke(null, elements); + return true; + } + + static bool Decode(PyObject tuple, out object value) + { + long itemCount = Runtime.PyTuple_Size(tuple.Handle); + if (itemCount == 0) + { + value = EmptyTuple; + return true; + } + var elements = new object[itemCount]; + var itemTypes = new Type[itemCount]; + value = null; + for (int itemIndex = 0; itemIndex < elements.Length; itemIndex++) + { + var pyItem = Runtime.PyTuple_GetItem(tuple.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, typeof(object), out elements[itemIndex], setError: false)) + { + return false; + } + + itemTypes[itemIndex] = elements[itemIndex]?.GetType() ?? typeof(object); + } + + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = factory.Invoke(null, elements); + return true; + } + + static readonly MethodInfo[] tupleCreate = + typeof(TTuple).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Tuple.Create)) + .OrderBy(m => m.GetParameters().Length) + .ToArray(); + + static readonly object EmptyTuple = tupleCreate[0].Invoke(null, parameters: new object[0]); + + public static void Register() + { + PyObjectConversions.RegisterEncoder(Instance); + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 0c2f912de..4686d0b3c 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -76,6 +76,8 @@ + + @@ -172,4 +174,4 @@ - + \ No newline at end of file diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index e7e047419..881b32cc9 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -135,8 +135,16 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + var encoded = PyObjectConversions.TryEncode(value, type); + if (encoded != null) { + Runtime.XIncref(encoded.Handle); + return encoded.Handle; + } + } + + if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -437,9 +445,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + TypeCode typeCode = Type.GetTypeCode(obType); + if (typeCode == TypeCode.Object) + { + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + return ToPrimitive(value, obType, out result, setError); } + internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + /// /// Convert a Python value to an instance of a primitive managed type. /// diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs new file mode 100644 index 000000000..a6ae91d50 --- /dev/null +++ b/src/runtime/converterextensions.cs @@ -0,0 +1,185 @@ +namespace Python.Runtime +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + /// + /// Defines conversion to CLR types (unmarshalling) + /// + public interface IPyObjectDecoder + { + /// + /// Checks if this decoder can decode from to + /// + bool CanDecode(PyObject objectType, Type targetType); + /// + /// Attempts do decode into a variable of specified type + /// + /// CLR type to decode into + /// Object to decode + /// The variable, that will receive decoding result + /// + bool TryDecode(PyObject pyObj, out T value); + } + + /// + /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) + /// + public interface IPyObjectEncoder + { + /// + /// Checks if encoder can encode CLR objects of specified type + /// + bool CanEncode(Type type); + /// + /// Attempts to encode CLR object into Python object + /// + PyObject TryEncode(object value); + } + + /// + /// This class allows to register additional marshalling codecs. + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static class PyObjectConversions + { + static readonly List decoders = new List(); + static readonly List encoders = new List(); + + /// + /// Registers specified encoder (marshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterEncoder(IPyObjectEncoder encoder) + { + if (encoder == null) throw new ArgumentNullException(nameof(encoder)); + + lock (encoders) + { + encoders.Add(encoder); + } + } + + /// + /// Registers specified decoder (unmarshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterDecoder(IPyObjectDecoder decoder) + { + if (decoder == null) throw new ArgumentNullException(nameof(decoder)); + + lock (decoders) + { + decoders.Add(decoder); + } + } + + #region Encoding + internal static PyObject TryEncode(object obj, Type type) + { + if (obj == null) throw new ArgumentNullException(nameof(obj)); + if (type == null) throw new ArgumentNullException(nameof(type)); + + foreach (var encoder in clrToPython.GetOrAdd(type, GetEncoders)) + { + var result = encoder.TryEncode(obj); + if (result != null) return result; + } + + return null; + } + + static readonly ConcurrentDictionary + clrToPython = new ConcurrentDictionary(); + static IPyObjectEncoder[] GetEncoders(Type type) + { + lock (encoders) + { + return encoders.Where(encoder => encoder.CanEncode(type)).ToArray(); + } + } + #endregion + + #region Decoding + static readonly ConcurrentDictionary + pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) + { + if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); + if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (targetType == null) throw new ArgumentNullException(nameof(targetType)); + + var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + result = null; + if (decoder == null) return false; + return decoder.Invoke(pyHandle, out result); + } + + static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) + { + IPyObjectDecoder decoder; + using (var pyType = new PyObject(sourceType)) + { + lock (decoders) + { + decoder = decoders.Find(d => d.CanDecode(pyType, targetType)); + if (decoder == null) return null; + } + } + + var decode = genericDecode.MakeGenericMethod(targetType); + + bool TryDecode(IntPtr pyHandle, out object result) + { + using (var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle))) + { + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + result = @params[1]; + return success; + } + } + + return TryDecode; + } + + static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); + + #endregion + + internal static void Reset() + { + lock (encoders) + lock (decoders) + { + clrToPython.Clear(); + pythonToClr.Clear(); + encoders.Clear(); + decoders.Clear(); + } + } + + struct TypePair : IEquatable + { + internal readonly IntPtr PyType; + internal readonly Type ClrType; + + public TypePair(IntPtr pyType, Type clrType) + { + this.PyType = pyType; + this.ClrType = clrType; + } + + public override int GetHashCode() + => this.ClrType.GetHashCode() ^ this.PyType.GetHashCode(); + + public bool Equals(TypePair other) + => this.PyType == other.PyType && this.ClrType == other.ClrType; + + public override bool Equals(object obj) => obj is TypePair other && this.Equals(other); + } + } +} diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 5073067d3..abe0abfa8 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -328,6 +328,8 @@ public static void Shutdown() ExecuteShutdownHandlers(); + PyObjectConversions.Reset(); + initialized = false; } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7748bafa9..f6500309c 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -603,6 +603,15 @@ internal static unsafe void XIncref(IntPtr op) #endif } + /// + /// Increase Python's ref counter for the given object, and get the object back. + /// + internal static IntPtr SelfIncRef(IntPtr op) + { + XIncref(op); + return op; + } + internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD From 6eca16943968219c42337757b45564f1dd8e6cc8 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 16 Jan 2020 11:04:06 -0800 Subject: [PATCH 02/11] fixed ConversionsObject test failing due to sequence to array conversion taking priority --- src/runtime/converter.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 881b32cc9..85f4fecb5 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -385,6 +385,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToPrimitive(value, doubleType, out result, setError); } + // give custom codecs a chance to take over conversion of sequences + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + if (Runtime.PySequence_Check(value)) { return ToArray(value, typeof(object[]), out result, setError); From 97e33c74482feb8421151c6902ff988e8aba16c7 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 16 Jan 2020 11:29:40 -0800 Subject: [PATCH 03/11] attempt to fix CI build issue with ValueTuple under Mono --- pythonnet.15.sln | 32 +++++++++++++++++++++ src/embed_tests/Python.EmbeddingTest.csproj | 3 ++ 2 files changed, 35 insertions(+) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index 096dfbe9a..a1e738900 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -19,6 +19,26 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F .editorconfig = .editorconfig EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" + ProjectSection(SolutionItems) = preProject + .travis.yml = .travis.yml + appveyor.yml = appveyor.yml + ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 + ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" + ProjectSection(SolutionItems) = preProject + setup.py = setup.py + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}" + ProjectSection(SolutionItems) = preProject + conda.recipe\bld.bat = conda.recipe\bld.bat + conda.recipe\meta.yaml = conda.recipe\meta.yaml + conda.recipe\README.md = conda.recipe\README.md + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -320,11 +340,17 @@ Global {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.Build.0 = DebugMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.Build.0 = DebugMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU @@ -344,11 +370,17 @@ Global {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.Build.0 = ReleaseMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.Build.0 = ReleaseMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index d8488011a..9ef1db170 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -73,6 +73,9 @@ ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + ..\..\packages\System.ValueTuple.4.5.0\lib\portable-net40+sl4+win8+wp8\System.ValueTuple.dll + From 449338f0196aa9ba4e8846537cb22a4cb6948019 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 30 Jan 2020 08:53:50 -0800 Subject: [PATCH 04/11] added RefereneAssemblies package reference to fix CI build --- src/perf_tests/Python.PerformanceTests.csproj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index 33949fdc1..4e1d28dcc 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -1,4 +1,4 @@ - + net461 @@ -12,6 +12,10 @@ + + all + runtime; build; native; contentfiles; analyzers + compile From 39b2347e10b11f23747d45f9e0f2c4a0624d61b7 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 31 Jan 2020 20:35:05 -0800 Subject: [PATCH 05/11] marked the new codecs API as unstable --- src/runtime/Codecs/TupleCodecs.cs | 1 + src/runtime/Util.cs | 5 ++++- src/runtime/converterextensions.cs | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs index 7c01eee46..f6bcc3fc9 100644 --- a/src/runtime/Codecs/TupleCodecs.cs +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -5,6 +5,7 @@ namespace Python.Runtime.Codecs using System.Linq; using System.Reflection; + [Obsolete(Util.UnstableApiMessage)] public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder { TupleCodec() { } diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..16d82fe6e 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -5,6 +5,9 @@ namespace Python.Runtime { internal class Util { + internal const string UnstableApiMessage = + "This API is unstable, and might be changed or removed in the next minor release"; + internal static Int64 ReadCLong(IntPtr tp, int offset) { // On Windows, a C long is always 32 bits. @@ -30,4 +33,4 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags) } } } -} \ No newline at end of file +} diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index a6ae91d50..0d7f0aff2 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// /// Defines conversion to CLR types (unmarshalling) /// + [Obsolete(Util.UnstableApiMessage)] public interface IPyObjectDecoder { /// @@ -28,6 +29,7 @@ public interface IPyObjectDecoder /// /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) /// + [Obsolete(Util.UnstableApiMessage)] public interface IPyObjectEncoder { /// @@ -44,6 +46,7 @@ public interface IPyObjectEncoder /// This class allows to register additional marshalling codecs. /// Python.NET will pick suitable encoder/decoder registered first /// + [Obsolete(Util.UnstableApiMessage)] public static class PyObjectConversions { static readonly List decoders = new List(); From daa2901b13233df80d92c25b474640ac0fd37e3d Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 31 Jan 2020 22:07:41 -0800 Subject: [PATCH 06/11] attempt to fix PyScopeTest.TestThread() reading stale value from res in Python 2.7 x64 --- src/embed_tests/TestPyScope.cs | 7 ++++++- src/runtime/assemblymanager.cs | 11 ++++++++++- src/runtime/moduleobject.cs | 13 +++---------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 21c0d2b3f..7a4aa0228 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using NUnit.Framework; using Python.Runtime; @@ -337,9 +338,12 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( + "import clr\n" + + "from System.Threading import Thread\n" + "def update():\n" + " global res, th_cnt\n" + " res += bb + 1\n" + + " Thread.MemoryBarrier()\n" + " th_cnt += 1\n" ); } @@ -364,8 +368,9 @@ public void TestThread() { cnt = ps.Get("th_cnt"); } - System.Threading.Thread.Sleep(10); + Thread.Sleep(10); } + Thread.MemoryBarrier(); using (Py.GIL()) { var result = ps.Get("res"); diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..f541caa9f 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -452,6 +452,7 @@ public static List GetNames(string nsname) /// looking in the currently loaded assemblies for the named /// type. Returns null if the named type cannot be found. /// + [Obsolete("Use LookupTypes and handle name conflicts")] public static Type LookupType(string qname) { foreach (Assembly assembly in assemblies) @@ -465,6 +466,14 @@ public static Type LookupType(string qname) return null; } + /// + /// Returns the objects for the given qualified name, + /// looking in the currently loaded assemblies for the named + /// type. + /// + public static IEnumerable LookupTypes(string qualifiedName) + => assemblies.Select(assembly => assembly.GetType(qualifiedName)).Where(type => type != null); + internal static Type[] GetTypes(Assembly a) { if (a.IsDynamic) @@ -492,4 +501,4 @@ internal static Type[] GetTypes(Assembly a) } } } -} \ No newline at end of file +} diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 544f69c81..d2397bfc6 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -105,13 +106,9 @@ public ManagedType GetAttribute(string name, bool guess) // Look for a type in the current namespace. Note that this // includes types, delegates, enums, interfaces and structs. // Only public namespace members are exposed to Python. - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -131,13 +128,9 @@ public ManagedType GetAttribute(string name, bool guess) return m; } - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; From ec982097e21ff7bddf8e78aa8b3d9b7662f8f70a Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 20 Feb 2020 23:25:31 -0800 Subject: [PATCH 07/11] fixed bad IncRef after PyTuple_New --- src/runtime/Codecs/TupleCodecs.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs index f6bcc3fc9..51c08cd3d 100644 --- a/src/runtime/Codecs/TupleCodecs.cs +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -12,8 +12,12 @@ public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder public static TupleCodec Instance { get; } = new TupleCodec(); public bool CanEncode(Type type) - => type.Namespace == typeof(TTuple).Namespace && type.Name.StartsWith(typeof(TTuple).Name + '`') - || type == typeof(object) || type == typeof(TTuple); + { + if (type == typeof(object) || type == typeof(TTuple)) return true; + return type.Namespace == typeof(TTuple).Namespace + // generic versions of tuples are named Tuple`TYPE_ARG_COUNT + && type.Name.StartsWith(typeof(TTuple).Name + '`'); + } public PyObject TryEncode(object value) { @@ -36,7 +40,7 @@ public PyObject TryEncode(object value) Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); fieldIndex++; } - return new PyTuple(Runtime.SelfIncRef(tuple)); + return new PyTuple(tuple); } public bool CanDecode(PyObject objectType, Type targetType) From 50a3822bf49f6e99a0b96a78b89af26fd32e0fcc Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 20 Feb 2020 23:31:54 -0800 Subject: [PATCH 08/11] corrected reference counting in Codecs --- src/runtime/converter.cs | 6 ++++-- src/runtime/converterextensions.cs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 85f4fecb5..a56fa88bb 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -138,8 +138,10 @@ internal static IntPtr ToPython(object value, Type type) if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { var encoded = PyObjectConversions.TryEncode(value, type); if (encoded != null) { - Runtime.XIncref(encoded.Handle); - return encoded.Handle; + result = encoded.Handle; + Runtime.XIncref(result); + encoded.Dispose(); + return result; } } diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index 0d7f0aff2..ce8ec2679 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -124,7 +124,7 @@ internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) { IPyObjectDecoder decoder; - using (var pyType = new PyObject(sourceType)) + using (var pyType = new PyObject(Runtime.SelfIncRef(sourceType))) { lock (decoders) { From 2e19f2c3d3a40502968a77149f4a62a590fdb9a3 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 21 Feb 2020 09:29:59 -0800 Subject: [PATCH 09/11] don't dispose encoded object in case codec keeps a cache of them --- src/runtime/converter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index a56fa88bb..7c53bdcb1 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -140,7 +140,6 @@ internal static IntPtr ToPython(object value, Type type) if (encoded != null) { result = encoded.Handle; Runtime.XIncref(result); - encoded.Dispose(); return result; } } From 399ae545c9c6a4ab0a0f1713e41eb1c41c889ca4 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 23 Feb 2020 15:10:22 -0800 Subject: [PATCH 10/11] do not dispose object, that might have been just decoded succesfully, as the decoded object might store a reference to the original PyObject --- src/runtime/converterextensions.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index ce8ec2679..fd012c6e4 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -137,13 +137,16 @@ static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type bool TryDecode(IntPtr pyHandle, out object result) { - using (var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle))) + var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle)); + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + if (!success) { - var @params = new object[] { pyObj, null }; - bool success = (bool)decode.Invoke(decoder, @params); - result = @params[1]; - return success; + pyObj.Dispose(); } + + result = @params[1]; + return success; } return TryDecode; From e2d333361c036243cec3c9aecab514d3888d8efa Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 23 Feb 2020 15:11:40 -0800 Subject: [PATCH 11/11] remove incref for tuple fields, as Converter.ToPython is supposed to return a new reference --- src/runtime/Codecs/TupleCodecs.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs index 51c08cd3d..4c81cac0b 100644 --- a/src/runtime/Codecs/TupleCodecs.cs +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -36,7 +36,6 @@ public PyObject TryEncode(object value) { var item = field.GetValue(value); IntPtr pyItem = Converter.ToPython(item); - Runtime.XIncref(pyItem); Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); fieldIndex++; }