diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..600215cf0 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,86 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 126bc5b63..d8952a3a9 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 6 + 7.3 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT @@ -77,6 +77,7 @@ + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index ffdbcbabe..7a0964a8a 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -14,7 +14,7 @@ 1591 ..\..\ $(SolutionDir)\bin\ - 6 + 7.3 true prompt @@ -73,6 +73,9 @@ ..\..\packages\NUnit.3.12.0\lib\net40\nunit.framework.dll + + ..\..\packages\System.ValueTuple.4.5.0\lib\portable-net40+sl4+win8+wp8\System.ValueTuple.dll + @@ -80,6 +83,7 @@ + diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 21c0d2b3f..7a4aa0228 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using NUnit.Framework; using Python.Runtime; @@ -337,9 +338,12 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( + "import clr\n" + + "from System.Threading import Thread\n" + "def update():\n" + " global res, th_cnt\n" + " res += bb + 1\n" + + " Thread.MemoryBarrier()\n" + " th_cnt += 1\n" ); } @@ -364,8 +368,9 @@ public void TestThread() { cnt = ps.Get("th_cnt"); } - System.Threading.Thread.Sleep(10); + Thread.Sleep(10); } + Thread.MemoryBarrier(); using (Py.GIL()) { var result = ps.Get("res"); diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 7a3fb37c4..590eaef8c 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - + + diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs new file mode 100644 index 000000000..4c81cac0b --- /dev/null +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -0,0 +1,133 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + [Obsolete(Util.UnstableApiMessage)] + public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder + { + TupleCodec() { } + public static TupleCodec Instance { get; } = new TupleCodec(); + + public bool CanEncode(Type type) + { + if (type == typeof(object) || type == typeof(TTuple)) return true; + return type.Namespace == typeof(TTuple).Namespace + // generic versions of tuples are named Tuple`TYPE_ARG_COUNT + && type.Name.StartsWith(typeof(TTuple).Name + '`'); + } + + public PyObject TryEncode(object value) + { + if (value == null) return null; + + var tupleType = value.GetType(); + if (tupleType == typeof(object)) return null; + if (!this.CanEncode(tupleType)) return null; + if (tupleType == typeof(TTuple)) return new PyTuple(); + + long fieldCount = tupleType.GetGenericArguments().Length; + var tuple = Runtime.PyTuple_New(fieldCount); + Exceptions.ErrorCheck(tuple); + int fieldIndex = 0; + foreach (FieldInfo field in tupleType.GetFields()) + { + var item = field.GetValue(value); + IntPtr pyItem = Converter.ToPython(item); + Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); + fieldIndex++; + } + return new PyTuple(tuple); + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == Runtime.PyTupleType && this.CanEncode(targetType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + value = default; + + if (!Runtime.PyTuple_Check(pyObj.Handle)) return false; + + if (typeof(T) == typeof(object)) + { + bool converted = Decode(pyObj, out object result); + if (converted) + { + value = (T)result; + return true; + } + + return false; + } + + var itemTypes = typeof(T).GetGenericArguments(); + long itemCount = Runtime.PyTuple_Size(pyObj.Handle); + if (itemTypes.Length != itemCount) return false; + + if (itemCount == 0) + { + value = (T)EmptyTuple; + return true; + } + + var elements = new object[itemCount]; + for (int itemIndex = 0; itemIndex < itemTypes.Length; itemIndex++) + { + IntPtr pyItem = Runtime.PyTuple_GetItem(pyObj.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, itemTypes[itemIndex], out elements[itemIndex], setError: false)) + { + return false; + } + } + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = (T)factory.Invoke(null, elements); + return true; + } + + static bool Decode(PyObject tuple, out object value) + { + long itemCount = Runtime.PyTuple_Size(tuple.Handle); + if (itemCount == 0) + { + value = EmptyTuple; + return true; + } + var elements = new object[itemCount]; + var itemTypes = new Type[itemCount]; + value = null; + for (int itemIndex = 0; itemIndex < elements.Length; itemIndex++) + { + var pyItem = Runtime.PyTuple_GetItem(tuple.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, typeof(object), out elements[itemIndex], setError: false)) + { + return false; + } + + itemTypes[itemIndex] = elements[itemIndex]?.GetType() ?? typeof(object); + } + + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = factory.Invoke(null, elements); + return true; + } + + static readonly MethodInfo[] tupleCreate = + typeof(TTuple).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Tuple.Create)) + .OrderBy(m => m.GetParameters().Length) + .ToArray(); + + static readonly object EmptyTuple = tupleCreate[0].Invoke(null, parameters: new object[0]); + + public static void Register() + { + PyObjectConversions.RegisterEncoder(Instance); + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index c79afee3e..1d40c2a38 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -76,6 +76,8 @@ + + @@ -175,4 +177,4 @@ - + \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index 29a5170ab..eb21cddbb 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -5,6 +5,9 @@ namespace Python.Runtime { internal static class Util { + internal const string UnstableApiMessage = + "This API is unstable, and might be changed or removed in the next minor release"; + internal static Int64 ReadCLong(IntPtr tp, int offset) { // On Windows, a C long is always 32 bits. diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 9d0296d47..e4ddaf915 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -452,6 +452,7 @@ public static List GetNames(string nsname) /// looking in the currently loaded assemblies for the named /// type. Returns null if the named type cannot be found. /// + [Obsolete("Use LookupTypes and handle name conflicts")] public static Type LookupType(string qname) { foreach (Assembly assembly in assemblies) @@ -465,6 +466,14 @@ public static Type LookupType(string qname) return null; } + /// + /// Returns the objects for the given qualified name, + /// looking in the currently loaded assemblies for the named + /// type. + /// + public static IEnumerable LookupTypes(string qualifiedName) + => assemblies.Select(assembly => assembly.GetType(qualifiedName)).Where(type => type != null); + internal static Type[] GetTypes(Assembly a) { if (a.IsDynamic) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index e7e047419..7c53bdcb1 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -135,8 +135,17 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + var encoded = PyObjectConversions.TryEncode(value, type); + if (encoded != null) { + result = encoded.Handle; + Runtime.XIncref(result); + return result; + } + } + + if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -377,6 +386,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToPrimitive(value, doubleType, out result, setError); } + // give custom codecs a chance to take over conversion of sequences + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + if (Runtime.PySequence_Check(value)) { return ToArray(value, typeof(object[]), out result, setError); @@ -437,9 +453,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + TypeCode typeCode = Type.GetTypeCode(obType); + if (typeCode == TypeCode.Object) + { + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + return ToPrimitive(value, obType, out result, setError); } + internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + /// /// Convert a Python value to an instance of a primitive managed type. /// diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs new file mode 100644 index 000000000..fd012c6e4 --- /dev/null +++ b/src/runtime/converterextensions.cs @@ -0,0 +1,191 @@ +namespace Python.Runtime +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + /// + /// Defines conversion to CLR types (unmarshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectDecoder + { + /// + /// Checks if this decoder can decode from to + /// + bool CanDecode(PyObject objectType, Type targetType); + /// + /// Attempts do decode into a variable of specified type + /// + /// CLR type to decode into + /// Object to decode + /// The variable, that will receive decoding result + /// + bool TryDecode(PyObject pyObj, out T value); + } + + /// + /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectEncoder + { + /// + /// Checks if encoder can encode CLR objects of specified type + /// + bool CanEncode(Type type); + /// + /// Attempts to encode CLR object into Python object + /// + PyObject TryEncode(object value); + } + + /// + /// This class allows to register additional marshalling codecs. + /// Python.NET will pick suitable encoder/decoder registered first + /// + [Obsolete(Util.UnstableApiMessage)] + public static class PyObjectConversions + { + static readonly List decoders = new List(); + static readonly List encoders = new List(); + + /// + /// Registers specified encoder (marshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterEncoder(IPyObjectEncoder encoder) + { + if (encoder == null) throw new ArgumentNullException(nameof(encoder)); + + lock (encoders) + { + encoders.Add(encoder); + } + } + + /// + /// Registers specified decoder (unmarshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterDecoder(IPyObjectDecoder decoder) + { + if (decoder == null) throw new ArgumentNullException(nameof(decoder)); + + lock (decoders) + { + decoders.Add(decoder); + } + } + + #region Encoding + internal static PyObject TryEncode(object obj, Type type) + { + if (obj == null) throw new ArgumentNullException(nameof(obj)); + if (type == null) throw new ArgumentNullException(nameof(type)); + + foreach (var encoder in clrToPython.GetOrAdd(type, GetEncoders)) + { + var result = encoder.TryEncode(obj); + if (result != null) return result; + } + + return null; + } + + static readonly ConcurrentDictionary + clrToPython = new ConcurrentDictionary(); + static IPyObjectEncoder[] GetEncoders(Type type) + { + lock (encoders) + { + return encoders.Where(encoder => encoder.CanEncode(type)).ToArray(); + } + } + #endregion + + #region Decoding + static readonly ConcurrentDictionary + pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) + { + if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); + if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (targetType == null) throw new ArgumentNullException(nameof(targetType)); + + var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + result = null; + if (decoder == null) return false; + return decoder.Invoke(pyHandle, out result); + } + + static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) + { + IPyObjectDecoder decoder; + using (var pyType = new PyObject(Runtime.SelfIncRef(sourceType))) + { + lock (decoders) + { + decoder = decoders.Find(d => d.CanDecode(pyType, targetType)); + if (decoder == null) return null; + } + } + + var decode = genericDecode.MakeGenericMethod(targetType); + + bool TryDecode(IntPtr pyHandle, out object result) + { + var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle)); + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + if (!success) + { + pyObj.Dispose(); + } + + result = @params[1]; + return success; + } + + return TryDecode; + } + + static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); + + #endregion + + internal static void Reset() + { + lock (encoders) + lock (decoders) + { + clrToPython.Clear(); + pythonToClr.Clear(); + encoders.Clear(); + decoders.Clear(); + } + } + + struct TypePair : IEquatable + { + internal readonly IntPtr PyType; + internal readonly Type ClrType; + + public TypePair(IntPtr pyType, Type clrType) + { + this.PyType = pyType; + this.ClrType = clrType; + } + + public override int GetHashCode() + => this.ClrType.GetHashCode() ^ this.PyType.GetHashCode(); + + public bool Equals(TypePair other) + => this.PyType == other.PyType && this.ClrType == other.ClrType; + + public override bool Equals(object obj) => obj is TypePair other && this.Equals(other); + } + } +} diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 5e7059b53..0a3933005 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -105,13 +106,9 @@ public ManagedType GetAttribute(string name, bool guess) // Look for a type in the current namespace. Note that this // includes types, delegates, enums, interfaces and structs. // Only public namespace members are exposed to Python. - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -131,13 +128,9 @@ public ManagedType GetAttribute(string name, bool guess) return m; } - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index a2da04af3..d5492ebb9 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -328,6 +328,8 @@ public static void Shutdown() ExecuteShutdownHandlers(); + PyObjectConversions.Reset(); + initialized = false; } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 6d75e4bef..963c9f475 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -603,6 +603,15 @@ internal static unsafe void XIncref(IntPtr op) #endif } + /// + /// Increase Python's ref counter for the given object, and get the object back. + /// + internal static IntPtr SelfIncRef(IntPtr op) + { + XIncref(op); + return op; + } + internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD