diff --git a/CHANGELOG.md b/CHANGELOG.md index a332f057d..60d516488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine +- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) ### Changed - Drop support for Python 2, 3.4, and 3.5 @@ -32,6 +33,7 @@ details about the cause of the failure - floating point values passed from Python are no longer silently truncated when .NET expects an integer [#1342][i1342] - More specific error messages for method argument mismatch +- BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. ### Fixed @@ -48,8 +50,8 @@ when .NET expects an integer [#1342][i1342] - Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions - Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects - Fixed objects returned by enumerating `PyObject` being disposed too soon -- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException -- `import` may now raise errors with more detail than "No module named X" +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException ([#1325][i1325]) +- `import` may now raise errors with more detail than "No module named X" - Providing an invalid type parameter to a generic type or method produces a helpful Python error ### Removed diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 0f263c721..54124ad34 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -338,9 +338,9 @@ internal static bool ToManagedValue(IntPtr value, Type obType, if (mt != null) { - if (mt is CLRObject) + if (mt is CLRObject co) { - object tmp = ((CLRObject)mt).inst; + object tmp = co.inst; if (obType.IsInstanceOfType(tmp)) { result = tmp; @@ -348,13 +348,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (setError) { - Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); + string typeString = tmp is null ? "null" : tmp.GetType().ToString(); + Exceptions.SetError(Exceptions.TypeError, $"{typeString} value cannot be converted to {obType}"); } return false; } - if (mt is ClassBase) + if (mt is ClassBase cb) { - var cb = (ClassBase)mt; if (!cb.type.Valid) { Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 3e6541c44..0a848904a 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -1,7 +1,9 @@ using System; -using System.Collections; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Reflection.Emit; +using System.Text; namespace Python.Runtime { @@ -11,23 +13,20 @@ namespace Python.Runtime /// internal class DelegateManager { - private Hashtable cache; - private Type basetype; - private Type listtype; - private Type voidtype; - private Type typetype; - private Type ptrtype; - private CodeGenerator codeGenerator; + private readonly Dictionary cache = new Dictionary(); + private readonly Type basetype = typeof(Dispatcher); + private readonly Type arrayType = typeof(object[]); + private readonly Type voidtype = typeof(void); + private readonly Type typetype = typeof(Type); + private readonly Type ptrtype = typeof(IntPtr); + private readonly CodeGenerator codeGenerator = new CodeGenerator(); + private readonly ConstructorInfo arrayCtor; + private readonly MethodInfo dispatch; public DelegateManager() { - basetype = typeof(Dispatcher); - listtype = typeof(ArrayList); - voidtype = typeof(void); - typetype = typeof(Type); - ptrtype = typeof(IntPtr); - cache = new Hashtable(); - codeGenerator = new CodeGenerator(); + arrayCtor = arrayType.GetConstructor(new[] { typeof(int) }); + dispatch = basetype.GetMethod("Dispatch"); } ///

@@ -58,10 +57,9 @@ private Type GetDispatcher(Type dtype) // unique signatures rather than delegate types, since multiple // delegate types with the same sig could use the same dispatcher. - object item = cache[dtype]; - if (item != null) + if (cache.TryGetValue(dtype, out Type item)) { - return (Type)item; + return item; } string name = $"__{dtype.FullName}Dispatcher"; @@ -103,34 +101,77 @@ private Type GetDispatcher(Type dtype) MethodBuilder mb = tb.DefineMethod("Invoke", MethodAttributes.Public, method.ReturnType, signature); - ConstructorInfo ctor = listtype.GetConstructor(Type.EmptyTypes); - MethodInfo dispatch = basetype.GetMethod("Dispatch"); - MethodInfo add = listtype.GetMethod("Add"); - il = mb.GetILGenerator(); - il.DeclareLocal(listtype); - il.Emit(OpCodes.Newobj, ctor); + // loc_0 = new object[pi.Length] + il.DeclareLocal(arrayType); + il.Emit(OpCodes.Ldc_I4, pi.Length); + il.Emit(OpCodes.Newobj, arrayCtor); il.Emit(OpCodes.Stloc_0); + bool anyByRef = false; + for (var c = 0; c < signature.Length; c++) { Type t = signature[c]; il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + if (t.IsByRef) + { + // The argument is a pointer. We must dereference the pointer to get the value or object it points to. + t = t.GetElementType(); + if (t.IsValueType) + { + il.Emit(OpCodes.Ldobj, t); + } + else + { + il.Emit(OpCodes.Ldind_Ref); + } + anyByRef = true; + } + if (t.IsValueType) { il.Emit(OpCodes.Box, t); } - il.Emit(OpCodes.Callvirt, add); - il.Emit(OpCodes.Pop); + // args[c] = arg + il.Emit(OpCodes.Stelem_Ref); } il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Call, dispatch); + if (anyByRef) + { + // Dispatch() will have modified elements of the args list that correspond to out parameters. + for (var c = 0; c < signature.Length; c++) + { + Type t = signature[c]; + if (t.IsByRef) + { + t = t.GetElementType(); + // *arg = args[c] + il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); + il.Emit(OpCodes.Ldelem_Ref); + if (t.IsValueType) + { + il.Emit(OpCodes.Unbox_Any, t); + il.Emit(OpCodes.Stobj, t); + } + else + { + il.Emit(OpCodes.Stind_Ref); + } + } + } + } + if (method.ReturnType == voidtype) { il.Emit(OpCodes.Pop); @@ -218,7 +259,7 @@ public void Dispose() GC.SuppressFinalize(this); } - public object Dispatch(ArrayList args) + public object Dispatch(object[] args) { IntPtr gs = PythonEngine.AcquireLock(); object ob; @@ -235,7 +276,7 @@ public object Dispatch(ArrayList args) return ob; } - public object TrueDispatch(ArrayList args) + private object TrueDispatch(object[] args) { MethodInfo method = dtype.GetMethod("Invoke"); ParameterInfo[] pi = method.GetParameters(); @@ -259,20 +300,108 @@ public object TrueDispatch(ArrayList args) throw e; } - if (rtype == typeof(void)) + try { - return null; - } + int byRefCount = pi.Count(parameterInfo => parameterInfo.ParameterType.IsByRef); + if (byRefCount > 0) + { + // By symmetry with MethodBinder.Invoke, when there are out + // parameters we expect to receive a tuple containing + // the result, if any, followed by the out parameters. If there is only + // one out parameter and the return type of the method is void, + // we instead receive the out parameter as the result from Python. + + bool isVoid = rtype == typeof(void); + int tupleSize = byRefCount + (isVoid ? 0 : 1); + if (isVoid && byRefCount == 1) + { + // The return type is void and there is a single out parameter. + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (!Converter.ToManaged(op, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function did not return {t.GetElementType()} (the out parameter type)"); + throw new PythonException(); + } + args[i] = newArg; + break; + } + } + return null; + } + else if (Runtime.PyTuple_Check(op) && Runtime.PyTuple_Size(op) == tupleSize) + { + int index = isVoid ? 0 : 1; + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + IntPtr item = Runtime.PyTuple_GetItem(op, index++); + if (!Converter.ToManaged(item, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element {i} was not {t.GetElementType()} (the out parameter type)"); + throw new PythonException(); + } + args[i] = newArg; + } + } + if (isVoid) + { + return null; + } + IntPtr item0 = Runtime.PyTuple_GetItem(op, 0); + if (!Converter.ToManaged(item0, rtype, out object result0, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)"); + throw new PythonException(); + } + return result0; + } + else + { + string tpName = Runtime.PyObject_GetTypeName(op); + if (Runtime.PyTuple_Check(op)) + { + tpName += $" of size {Runtime.PyTuple_Size(op)}"; + } + StringBuilder sb = new StringBuilder(); + if (!isVoid) sb.Append(rtype.FullName); + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (sb.Length > 0) sb.Append(","); + sb.Append(t.GetElementType().FullName); + } + } + string returnValueString = isVoid ? "" : "the return value and "; + Exceptions.RaiseTypeError($"Expected a tuple ({sb}) of {returnValueString}the values for out and ref parameters, got {tpName}."); + throw new PythonException(); + } + } + + if (rtype == typeof(void)) + { + return null; + } - object result; - if (!Converter.ToManaged(op, rtype, out result, true)) + object result; + if (!Converter.ToManaged(op, rtype, out result, true)) + { + throw new PythonException(); + } + + return result; + } + finally { Runtime.XDecref(op); - throw new PythonException(); } - - Runtime.XDecref(op); - return result; } } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 3f879d3c4..034c1c3e8 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -960,34 +960,35 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i } // If there are out parameters, we return a tuple containing - // the result followed by the out parameters. If there is only + // the result, if any, followed by the out parameters. If there is only // one out parameter and the return type of the method is void, // we return the out parameter as the result to Python (for // code compatibility with ironpython). var mi = (MethodInfo)binding.info; - if (binding.outs == 1 && mi.ReturnType == typeof(void)) - { - } - if (binding.outs > 0) { ParameterInfo[] pi = mi.GetParameters(); int c = pi.Length; var n = 0; - IntPtr t = Runtime.PyTuple_New(binding.outs + 1); - IntPtr v = Converter.ToPython(result, mi.ReturnType); - Runtime.PyTuple_SetItem(t, n, v); - n++; + bool isVoid = mi.ReturnType == typeof(void); + int tupleSize = binding.outs + (isVoid ? 0 : 1); + IntPtr t = Runtime.PyTuple_New(tupleSize); + if (!isVoid) + { + IntPtr v = Converter.ToPython(result, mi.ReturnType); + Runtime.PyTuple_SetItem(t, n, v); + n++; + } for (var i = 0; i < c; i++) { Type pt = pi[i].ParameterType; - if (pi[i].IsOut || pt.IsByRef) + if (pt.IsByRef) { - v = Converter.ToPython(binding.args[i], pt.GetElementType()); + IntPtr v = Converter.ToPython(binding.args[i], pt.GetElementType()); Runtime.PyTuple_SetItem(t, n, v); n++; } @@ -995,7 +996,7 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding.outs == 1 && mi.ReturnType == typeof(void)) { - v = Runtime.PyTuple_GetItem(t, 1); + IntPtr v = Runtime.PyTuple_GetItem(t, 0); Runtime.XIncref(v); Runtime.XDecref(t); return v; diff --git a/src/testing/delegatetest.cs b/src/testing/delegatetest.cs index e2df9475f..ee66bdad7 100644 --- a/src/testing/delegatetest.cs +++ b/src/testing/delegatetest.cs @@ -13,6 +13,12 @@ namespace Python.Test public delegate bool BoolDelegate(); + public delegate void OutStringDelegate(out string value); + public delegate void RefStringDelegate(ref string value); + public delegate void OutIntDelegate(out int value); + public delegate void RefIntDelegate(ref int value); + public delegate void RefIntRefStringDelegate(ref int intValue, ref string stringValue); + public delegate int IntRefIntRefStringDelegate(ref int intValue, ref string stringValue); public class DelegateTest { @@ -27,6 +33,8 @@ public class DelegateTest public StringDelegate stringDelegate; public ObjectDelegate objectDelegate; public BoolDelegate boolDelegate; + public OutStringDelegate outStringDelegate; + public RefStringDelegate refStringDelegate; public DelegateTest() { @@ -42,6 +50,11 @@ public static string StaticSayHello() return "hello"; } + public void OutHello(out string value) + { + value = "hello"; + } + public string CallStringDelegate(StringDelegate d) { return d(); @@ -56,5 +69,35 @@ public bool CallBoolDelegate(BoolDelegate d) { return d(); } + + public void CallOutIntDelegate(OutIntDelegate d, out int value) + { + d(out value); + } + + public void CallRefIntDelegate(RefIntDelegate d, ref int value) + { + d(ref value); + } + + public void CallOutStringDelegate(OutStringDelegate d, out string value) + { + d(out value); + } + + public void CallRefStringDelegate(RefStringDelegate d, ref string value) + { + d(ref value); + } + + public void CallRefIntRefStringDelegate(RefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + d(ref intValue, ref stringValue); + } + + public int CallIntRefIntRefStringDelegate(IntRefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + return d(ref intValue, ref stringValue); + } } } diff --git a/src/testing/eventtest.cs b/src/testing/eventtest.cs index dfbd5c881..c9573f71a 100644 --- a/src/testing/eventtest.cs +++ b/src/testing/eventtest.cs @@ -7,7 +7,6 @@ namespace Python.Test ///

public delegate void EventHandlerTest(object sender, EventArgsTest e); - #pragma warning disable 67 // Unused events, these are only accessed from Python public class EventTest { @@ -27,6 +26,10 @@ public class EventTest private event EventHandlerTest PrivateEvent; + public event OutStringDelegate OutStringEvent; + public event OutIntDelegate OutIntEvent; + public event RefStringDelegate RefStringEvent; + public event RefIntDelegate RefIntEvent; public static int s_value; public int value; @@ -77,6 +80,27 @@ protected static void OnProtectedStaticEvent(EventArgsTest e) } } + public void OnRefStringEvent(ref string data) + { + RefStringEvent?.Invoke(ref data); + } + + public void OnRefIntEvent(ref int data) + { + RefIntEvent?.Invoke(ref data); + } + + public void OnOutStringEvent(out string data) + { + data = default; + OutStringEvent?.Invoke(out data); + } + + public void OnOutIntEvent(out int data) + { + data = default; + OutIntEvent?.Invoke(out data); + } public void GenericHandler(object sender, EventArgsTest e) { @@ -88,6 +112,26 @@ public static void StaticHandler(object sender, EventArgsTest e) s_value = e.value; } + public void OutStringHandler(out string data) + { + data = value.ToString(); + } + + public void OutIntHandler(out int data) + { + data = value; + } + + public void RefStringHandler(ref string data) + { + data += "!"; + } + + public void RefIntHandler(ref int data) + { + data++; + } + public static void ShutUpCompiler() { // Quiet compiler warnings. diff --git a/src/tests/test_delegate.py b/src/tests/test_delegate.py index 909fd0f05..52ac8226d 100644 --- a/src/tests/test_delegate.py +++ b/src/tests/test_delegate.py @@ -276,6 +276,166 @@ def test_invalid_object_delegate(): with pytest.raises(TypeError): ob.CallObjectDelegate(d) +def test_out_int_delegate(): + """Test delegate with an out int parameter.""" + from Python.Test import OutIntDelegate + value = 7 + + def out_hello_func(ignored): + return 5 + + d = OutIntDelegate(out_hello_func) + result = d(value) + assert result == 5 + + ob = DelegateTest() + result = ob.CallOutIntDelegate(d, value) + assert result == 5 + + def invalid_handler(ignored): + return '5' + + d = OutIntDelegate(invalid_handler) + with pytest.raises(TypeError): + result = d(value) + +def test_out_string_delegate(): + """Test delegate with an out string parameter.""" + from Python.Test import OutStringDelegate + value = 'goodbye' + + def out_hello_func(ignored): + return 'hello' + + d = OutStringDelegate(out_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallOutStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefIntDelegate + value = 7 + + def ref_hello_func(data): + assert data == value + return data + 1 + + d = RefIntDelegate(ref_hello_func) + result = d(value) + assert result == value + 1 + + ob = DelegateTest() + result = ob.CallRefIntDelegate(d, value) + assert result == value + 1 + +def test_ref_string_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefStringDelegate + value = 'goodbye' + + def ref_hello_func(data): + assert data == value + return 'hello' + + d = RefStringDelegate(ref_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallRefStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import RefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + 1, stringValue + '!') + + d = RefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = RefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = RefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def long_tuple(intValue, stringValue): + return (5, 'a', 'b') + + d = RefIntRefStringDelegate(long_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_tuple_item(intValue, stringValue): + return ('a', 'b') + + d = RefIntRefStringDelegate(wrong_tuple_item) + with pytest.raises(TypeError): + result = d(intData, stringData) + +def test_int_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import IntRefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + len(stringValue), intValue + 1, stringValue + '!') + + d = IntRefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallIntRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = IntRefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = IntRefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_return_type(intValue, stringValue): + return ('a', 7, 'b') + + d = IntRefIntRefStringDelegate(wrong_return_type) + with pytest.raises(TypeError): + result = d(intData, stringData) + # test async delegates # test multicast delegates diff --git a/src/tests/test_event.py b/src/tests/test_event.py index e9c0ffd8a..885589032 100644 --- a/src/tests/test_event.py +++ b/src/tests/test_event.py @@ -295,6 +295,41 @@ def handler(sender, args, dict_=dict_): ob.OnPublicEvent(EventArgsTest(20)) assert dict_['value'] == 10 +def test_out_function_handler(): + """Test function handlers with Out arguments.""" + ob = EventTest() + + value = 10 + def handler(ignored): + return value + + ob.OutIntEvent += handler + result = ob.OnOutIntEvent(55) + assert result == value + + ob.OutStringEvent += handler + value = 'This is the event data' + result = ob.OnOutStringEvent('Hello') + assert result == value + +def test_ref_function_handler(): + """Test function handlers with Ref arguments.""" + ob = EventTest() + + value = 10 + def handler(data): + return value + data + + ob.RefIntEvent += ob.RefIntHandler + ob.RefIntEvent += handler + result = ob.OnRefIntEvent(5) + assert result == value + 5 + 1 + + ob.RefStringEvent += ob.RefStringHandler + ob.RefStringEvent += handler + value = 'This is the event data' + result = ob.OnRefStringEvent('!') + assert result == value + '!!' def test_add_non_callable_handler(): """Test handling of attempts to add non-callable handlers."""