From 36736e5c9ead23704a34031adcc50d8e09b20ad1 Mon Sep 17 00:00:00 2001 From: Gil Arasa Verge Date: Wed, 16 Dec 2020 15:26:20 -0500 Subject: [PATCH 1/2] Python async function calls --- .../include/py_loader/py_loader_impl.h | 2 +- .../loaders/py_loader/source/py_loader_impl.c | 426 ++++++++++++++++-- .../loaders/py_loader/source/py_loader_port.c | 2 +- source/tests/CMakeLists.txt | 1 + .../metacall_python_async_test/CMakeLists.txt | 153 +++++++ .../source/main.cpp | 28 ++ .../source/metacall_python_async_test.cpp | 149 ++++++ 7 files changed, 733 insertions(+), 28 deletions(-) create mode 100644 source/tests/metacall_python_async_test/CMakeLists.txt create mode 100644 source/tests/metacall_python_async_test/source/main.cpp create mode 100644 source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp diff --git a/source/loaders/py_loader/include/py_loader/py_loader_impl.h b/source/loaders/py_loader/include/py_loader/py_loader_impl.h index 1d64f61ca..55167d388 100644 --- a/source/loaders/py_loader/include/py_loader/py_loader_impl.h +++ b/source/loaders/py_loader/include/py_loader/py_loader_impl.h @@ -52,7 +52,7 @@ PY_LOADER_API int py_loader_impl_discover(loader_impl impl, loader_handle handle PY_LOADER_API int py_loader_impl_destroy(loader_impl impl); -PY_LOADER_NO_EXPORT type_id py_loader_impl_capi_to_value_type(PyObject * obj); +PY_LOADER_NO_EXPORT type_id py_loader_impl_capi_to_value_type(loader_impl impl, PyObject * obj); PY_LOADER_NO_EXPORT value py_loader_impl_capi_to_value(loader_impl impl, PyObject * obj, type_id id); diff --git a/source/loaders/py_loader/source/py_loader_impl.c b/source/loaders/py_loader/source/py_loader_impl.c index 964881f45..9e8ff3ea3 100644 --- a/source/loaders/py_loader/source/py_loader_impl.c +++ b/source/loaders/py_loader/source/py_loader_impl.c @@ -18,8 +18,8 @@ * */ -#include -#include +#include +#include #include #include @@ -46,6 +46,13 @@ typedef struct loader_impl_py_function_type } * loader_impl_py_function; +typedef struct loader_impl_py_future_type +{ + loader_impl_py py_impl; + PyObject *future; + +} * loader_impl_py_future; + typedef struct loader_impl_py_class_type { PyObject *class; @@ -83,6 +90,12 @@ struct loader_impl_py_type PyObject *traceback_module; PyObject *traceback_format_exception; + PyObject *asyncio_module; + PyObject *asyncio_isfuture; + PyObject *asyncio_iscoroutinefunction; + PyObject *asyncio_create_task; + PyObject *asyncio_get_event_loop; + #if (!defined(NDEBUG) || defined(DEBUG) || defined(_DEBUG) || defined(__DEBUG) || defined(__DEBUG__)) PyObject *gc_module; PyObject *gc_set_debug; @@ -99,6 +112,17 @@ typedef struct loader_impl_py_function_type_invoke_state_type } * loader_impl_py_function_type_invoke_state; +typedef struct loader_impl_py_await_invoke_callback_state_type +{ + loader_impl_py py_impl; + + function_resolve_callback resolve_callback; + function_reject_callback reject_callback; + void * context; + +} * loader_impl_py_await_invoke_callback_state; + + static void py_loader_impl_error_print(loader_impl_py py_impl); #if (!defined(NDEBUG) || defined(DEBUG) || defined(_DEBUG) || defined(__DEBUG) || defined(__DEBUG__)) @@ -221,6 +245,44 @@ int function_py_interface_create(function func, function_impl impl) } +int future_py_interface_create(future f, future_impl impl) +{ + (void)f; + (void)impl; + + // asyncio.loop.create_future + + return 0; +} + +void future_py_interface_destroy(future f, future_impl impl) +{ + loader_impl_py_future py_future = (loader_impl_py_future)impl; + + if (py_future != NULL) + { + loader_impl_py py_impl = py_future->py_impl; + + /* TODO */ + + (void)f; + (void)impl; + + free(py_future); + } +} + +future_interface future_py_singleton() +{ + static struct future_interface_type py_future_interface = + { + &future_py_interface_create, + &future_py_interface_destroy + }; + + return &py_future_interface; +} + int py_object_interface_create(object obj, object_impl impl) { (void)obj; @@ -246,7 +308,7 @@ value py_object_interface_get(object obj, object_impl impl, const char * key) PyObject * generic_attr = PyObject_GenericGetAttr(pyobject_object, key_py_str); Py_DECREF(key_py_str); - return py_loader_impl_capi_to_value(impl, generic_attr, py_loader_impl_capi_to_value_type(generic_attr)); + return py_loader_impl_capi_to_value(impl, generic_attr, py_loader_impl_capi_to_value_type(py_object->impl, generic_attr)); } int py_object_interface_set(object obj, object_impl impl, const char * key, value v) @@ -308,7 +370,7 @@ value py_object_interface_method_invoke(object obj, object_impl impl, const char return NULL; } - return py_loader_impl_capi_to_value(impl, python_object, py_loader_impl_capi_to_value_type(python_object)); + return py_loader_impl_capi_to_value(impl, python_object, py_loader_impl_capi_to_value_type(obj_impl->impl, python_object)); } value py_object_interface_method_await(object obj, object_impl impl, const char * key, object_args args, size_t size, object_resolve_callback resolve, object_reject_callback reject, void * ctx) @@ -438,7 +500,7 @@ value py_class_interface_static_get(klass cls, class_impl impl, const char * key PyObject * generic_attr = PyObject_GenericGetAttr(pyobject_class, key_py_str); Py_DECREF(key_py_str); - return py_loader_impl_capi_to_value(impl, generic_attr, py_loader_impl_capi_to_value_type(generic_attr)); + return py_loader_impl_capi_to_value(impl, generic_attr, py_loader_impl_capi_to_value_type(py_class->impl, generic_attr)); } int py_class_interface_static_set(klass cls, class_impl impl, const char * key, value v) @@ -503,7 +565,7 @@ value py_class_interface_static_invoke(klass cls, class_impl impl, const char * return NULL; } - return py_loader_impl_capi_to_value(impl, python_object, py_loader_impl_capi_to_value_type(python_object)); + return py_loader_impl_capi_to_value(impl, python_object, py_loader_impl_capi_to_value_type(cls_impl->impl, python_object)); } value py_class_interface_static_await(klass cls, class_impl impl, const char * key, class_args args, size_t size, class_resolve_callback resolve, class_reject_callback reject, void * ctx) @@ -552,8 +614,10 @@ class_interface py_class_interface_singleton(void) return &py_class_interface; } -type_id py_loader_impl_capi_to_value_type(PyObject *obj) +type_id py_loader_impl_capi_to_value_type(loader_impl impl, PyObject *obj) { + loader_impl_py py_impl = loader_impl_get(impl); + if (PyBool_Check(obj)) { return TYPE_BOOL; @@ -608,9 +672,14 @@ type_id py_loader_impl_capi_to_value_type(PyObject *obj) { return TYPE_NULL; } + else if (PyObject_CallFunctionObjArgs(py_impl->asyncio_isfuture, obj, NULL)) + { + return TYPE_FUTURE; + } else if (PyObject_IsSubclass(obj, (PyObject *)&PyBaseObject_Type) != 0) { /* TODO: This is based on trial and error and is not correct, but hey, it works! (for now) */ + /* TODO: Better to use inspect.isclass(obj) using py_impl->inspect_module */ /* PyObject_IsSubclass: if the class derived is identical to or derived from PyBaseObject_Type returns 1 */ /* in case of an error, returns -1 */ @@ -730,7 +799,7 @@ value py_loader_impl_capi_to_value(loader_impl impl, PyObject *obj, type_id id) PyObject *element = PyList_GetItem(obj, iterator); /* TODO: Review recursion overflow */ - array_value[iterator] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(element)); + array_value[iterator] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(impl, element)); } } else if (id == TYPE_MAP) @@ -808,7 +877,7 @@ value py_loader_impl_capi_to_value(loader_impl impl, PyObject *obj, type_id id) array_value[0] = value_create_string(key_str, (size_t)key_length); /* TODO: Review recursion overflow */ - array_value[1] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(element)); + array_value[1] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(impl, element)); ++key_iterator; } @@ -889,6 +958,29 @@ value py_loader_impl_capi_to_value(loader_impl impl, PyObject *obj, type_id id) { v = value_create_null(); } + else if (id == TYPE_FUTURE) + { + + loader_impl_py_future py_future = malloc(sizeof(struct loader_impl_py_future_type)); + + future f; + + if (py_future == NULL) + { + return NULL; + } + + f = future_create(py_future, &future_py_singleton); + + if (f == NULL) + { + free(py_future); + + return NULL; + } + + v = value_create_future(f); + } else if (id == TYPE_CLASS) { loader_impl_py_class py_cls = malloc(sizeof(struct loader_impl_py_class_type)); @@ -919,7 +1011,7 @@ value py_loader_impl_capi_to_value(loader_impl impl, PyObject *obj, type_id id) Py_INCREF(object_class); /* TODO: Will capi_to_value recognize and be able to parse a PyTypeObject ? */ - value obj_cls = py_loader_impl_capi_to_value(impl, (PyObject*)object_class, py_loader_impl_capi_to_value_type((PyObject*)object_class)); + value obj_cls = py_loader_impl_capi_to_value(impl, (PyObject*)object_class, py_loader_impl_capi_to_value_type(impl, (PyObject*)object_class)); /* Not using class_new() here because the object is already instantiated in the runtime */ /* So we must avoid calling it's constructor again */ @@ -1218,7 +1310,7 @@ function_return function_py_interface_invoke(function func, function_impl impl, if (ret_type == NULL) { - id = py_loader_impl_capi_to_value_type(result); + id = py_loader_impl_capi_to_value_type(impl, result); } else { @@ -1239,17 +1331,211 @@ function_return function_py_interface_invoke(function func, function_impl impl, return NULL; } +PyObject * task_callback_handler_impl(PyObject * self, PyObject * pyfuture) +{ + /* self will always be NULL */ + (void)self; + + log_write("metacall", LOG_LEVEL_DEBUG, "pyfuture <%x>", pyfuture); + + + PyObject * capsule = PyObject_GetAttrString(pyfuture, "__metacall__capsule"); + if(capsule == NULL) + { + log_write("metacall", LOG_LEVEL_ERROR, "Invalid python capsule"); + Py_RETURN_NONE; + } + + loader_impl_py_await_invoke_callback_state callback_state = PyCapsule_GetPointer(capsule, NULL); + Py_DECREF(capsule); + + /* pyfuture should never raise InvalidStateError exception here */ + /* because this is a callback set in Future.add_done_callback */ + + PyObject *result_str = PyUnicode_FromString("result"); + PyObject *result = PyObject_CallMethodObjArgs(pyfuture, result_str, NULL); + Py_DECREF(result_str); + + type_id id = py_loader_impl_capi_to_value_type((loader_impl)callback_state->py_impl, result); + value v = py_loader_impl_capi_to_value((loader_impl)callback_state->py_impl, result, id); + + Py_DECREF(result); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(callback_state->py_impl); + + /* Handle CancelledError or any exceptions raised by the user or propagate them up?? */ + + callback_state->reject_callback(v, callback_state->context); + } + else + { + value ret = callback_state->resolve_callback(v, callback_state->context); + + // value_to_capi(ret) + /* ^^ TODO: Take this return value and convert to PyObject and return it from THIS function */ + } + + free(callback_state); + + Py_RETURN_NONE; +} + function_return function_py_interface_await(function func, function_impl impl, function_args args, size_t size, function_resolve_callback resolve_callback, function_reject_callback reject_callback, void *context) { - /* TODO */ + loader_impl_py_function py_func = (loader_impl_py_function)impl; + signature s = function_signature(func); + const size_t args_size = size; + const size_t signature_args_size = signature_count(s); + type ret_type = signature_get_return(s); + PyObject *pytask = NULL; + size_t args_count; + loader_impl_py py_impl = loader_impl_get(py_func->impl); + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *tuple_args; + + /* Allocate dynamically more space for values in case of variable arguments */ + void **values = args_size > signature_args_size ? malloc(sizeof(void *) * args_size) : py_func->values; - (void)func; - (void)impl; - (void)args; - (void)size; - (void)resolve_callback; - (void)reject_callback; - (void)context; + tuple_args = PyTuple_New(args_size); + + for (args_count = 0; args_count < args_size; ++args_count) + { + type t = args_count < signature_args_size ? signature_get_type(s, args_count) : NULL; + + type_id id = TYPE_INVALID; + + if (t == NULL) + { + id = value_type_id((value)args[args_count]); + } + else + { + id = type_index(t); + } + + values[args_count] = py_loader_impl_value_to_capi(py_func->impl, id, args[args_count]); + + if (values[args_count] != NULL) + { + PyTuple_SetItem(tuple_args, args_count, values[args_count]); + } + } + + PyObject *coroutine = PyObject_CallObject(py_func->func, tuple_args); + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + PyGILState_Release(gstate); + return NULL; + } + + /* TODO. should call https://docs.python.org/3.6/library/asyncio-eventloop.html#asyncio.AbstractEventLoop.close ??*/ + PyObject *loop = PyObject_CallFunctionObjArgs(py_impl->asyncio_get_event_loop, NULL); + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + PyGILState_Release(gstate); + return NULL; + } + + pytask = PyObject_CallFunctionObjArgs(py_impl->asyncio_create_task, coroutine, loop, NULL); + + loader_impl_py_await_invoke_callback_state callback_state = malloc(sizeof(struct loader_impl_py_await_invoke_callback_state_type)); + callback_state->resolve_callback = resolve_callback; + callback_state->reject_callback = reject_callback; + callback_state->py_impl = py_impl; + callback_state->context = context; + + /* Use this new Task member attribute to handle the callback pointers to the python task callback */ + PyObject * callback_status = PyCapsule_New(callback_state, NULL, NULL); + if (callback_status != NULL && PyObject_SetAttrString(pytask, "__metacall__capsule", callback_status)) + { + PyGILState_Release(gstate); + return NULL; + } + + /* Wrap the C callback around a PyObject so that the interpreter knows how to call back as if it was python code */ + PyMethodDef py_task_callback_handler_def = { + "metacall_task_callback_handler", + task_callback_handler_impl, + METH_O, + "MetaCall async invoke callback"}; + PyObject *py_task_callback_handler = PyCFunction_NewEx(&py_task_callback_handler_def, NULL, NULL); + + /* When the future is done, automatically call the callback */ + PyObject *add_done_callback_str = PyUnicode_FromString("add_done_callback"); + PyObject_CallMethodObjArgs(pytask, add_done_callback_str, py_task_callback_handler, NULL); + Py_DECREF(add_done_callback_str); + + // TODO: ? Py_DECREF(py_task_callback_handler); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + PyGILState_Release(gstate); + return NULL; + } + + PyObject *run_forever_str = PyUnicode_FromString("run_forever"); + PyObject_CallMethodObjArgs(loop, run_forever_str, NULL); + Py_DECREF(run_forever_str); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + PyGILState_Release(gstate); + return NULL; + } + + /* Loop will stop after an iteration. Perhaps this is not correct and something like asyncio.run_until_complete */ + /* should be used instead although removing the last line of that function which calls to future.result() */ + PyObject *stop_str = PyUnicode_FromString("stop"); + PyObject_CallMethodObjArgs(loop, stop_str, NULL); + Py_DECREF(stop_str); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + PyGILState_Release(gstate); + return NULL; + } + + + Py_DECREF(tuple_args); + + /* Variable arguments */ + if (args_size > signature_args_size) + { + free(values); + } + + if (pytask != NULL) + { + value v = NULL; + + type_id id = TYPE_INVALID; + + if (ret_type == NULL) + { + id = py_loader_impl_capi_to_value_type((loader_impl)py_impl, pytask); + } + else + { + id = type_index(ret_type); + } + + v = py_loader_impl_capi_to_value(py_func->impl, pytask, id); + + Py_DECREF(pytask); + + PyGILState_Release(gstate); + + return v; + } + + PyGILState_Release(gstate); return NULL; } @@ -1347,7 +1633,7 @@ PyObject *py_loader_impl_function_type_invoke(PyObject *self, PyObject *args) { PyObject *arg = PyTuple_GetItem(args, (Py_ssize_t)args_count); - type_id id = py_loader_impl_capi_to_value_type(arg); + type_id id = py_loader_impl_capi_to_value_type(invoke_state->impl, arg); value_args[args_count] = py_loader_impl_capi_to_value(invoke_state->impl, arg, id); } @@ -1526,6 +1812,70 @@ int py_loader_impl_initialize_inspect(loader_impl impl, loader_impl_py py_impl) return 1; } +int py_loader_impl_initialize_asyncio_module(loader_impl impl, loader_impl_py py_impl) +{ + (void)impl; + + PyObject *module_name = PyUnicode_DecodeFSDefault("asyncio"); + + py_impl->asyncio_module = PyImport_Import(module_name); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + + Py_DECREF(module_name); + + return 1; + } + + if (py_impl->asyncio_module != NULL) + { + py_impl->asyncio_isfuture = PyObject_GetAttrString(py_impl->asyncio_module, "isfuture"); + + if (py_impl->asyncio_isfuture == NULL) + { + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.isfuture(obj)"); + return 1; + } + + py_impl->asyncio_iscoroutinefunction = PyObject_GetAttrString(py_impl->asyncio_module, "iscoroutinefunction"); + + if (py_impl->asyncio_iscoroutinefunction == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.iscoroutinefunction(func)"); + return 1; + } + + py_impl->asyncio_create_task = PyObject_GetAttrString(py_impl->asyncio_module, "run_coroutine_threadsafe"); + + if (py_impl->asyncio_create_task == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.run_coroutine_threadsafe(coro, loop)"); + return 1; + } + + py_impl->asyncio_get_event_loop = PyObject_GetAttrString(py_impl->asyncio_module, "get_event_loop"); + + if (py_impl->asyncio_get_event_loop == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_create_task); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.get_event_loop()"); + return 1; + } + } + + Py_DECREF(module_name); + + return 0; +} + + int py_loader_impl_initialize_traceback(loader_impl impl, loader_impl_py py_impl) { PyObject *module_name = PyUnicode_DecodeFSDefault("traceback"); @@ -1680,6 +2030,15 @@ loader_impl_data py_loader_impl_initialize(loader_impl impl, configuration confi return NULL; } + if (py_loader_impl_initialize_asyncio_module(impl, py_impl) != 0) + { + PyGILState_Release(gstate); + + free(py_impl); + + return NULL; + } + if (PY_LOADER_PORT_NAME_FUNC() == NULL) { PyGILState_Release(gstate); @@ -2079,6 +2438,15 @@ int py_loader_impl_discover_func(loader_impl impl, PyObject *func, function f) } } + if (PyObject_CallFunctionObjArgs(py_impl->asyncio_iscoroutinefunction, func, NULL)) + { + function_async(f, FUNCTION_ASYNC); + } + else + { + function_async(f, FUNCTION_SYNC); + } + signature_set_return(s, py_loader_impl_discover_type(impl, return_annotation)); return 0; @@ -2130,12 +2498,12 @@ static int py_loader_impl_discover_class(loader_impl impl, PyObject *obj, klass if (!PyUnicode_CompareWithASCIIString(tuple_key, "__dict__") || !PyUnicode_CompareWithASCIIString(tuple_key, "__weakref__")) continue; - // value key = py_loader_impl_capi_to_value(impl, tuple_key, py_loader_impl_capi_to_value_type(tuple_key)); - // value val = py_loader_impl_capi_to_value(impl, tuple_val, py_loader_impl_capi_to_value_type(tuple_val)); + // value key = py_loader_impl_capi_to_value(impl, tuple_key, py_loader_impl_capi_to_value_type(impl, tuple_key)); + // value val = py_loader_impl_capi_to_value(impl, tuple_val, py_loader_impl_capi_to_value_type(impl, tuple_val)); log_write("metacall", LOG_LEVEL_DEBUG, "Introspection: class member %s, type %s", PyUnicode_AsUTF8(tuple_key), - type_id_name(py_loader_impl_capi_to_value_type(tuple_val))); + type_id_name(py_loader_impl_capi_to_value_type(impl, tuple_val))); } } @@ -2152,12 +2520,12 @@ static int py_loader_impl_validate_object(loader_impl impl, PyObject *obj, objec while (PyDict_Next(PyObject_GetAttrString(obj, "__dict__"), &pos, &dict_key, &dict_val)) { - value attribute_key = py_loader_impl_capi_to_value(impl, dict_key, py_loader_impl_capi_to_value_type(dict_key)); - value attribute_val = py_loader_impl_capi_to_value(impl, dict_val, py_loader_impl_capi_to_value_type(dict_val)); + value attribute_key = py_loader_impl_capi_to_value(impl, dict_key, py_loader_impl_capi_to_value_type(impl, dict_key)); + value attribute_val = py_loader_impl_capi_to_value(impl, dict_val, py_loader_impl_capi_to_value_type(impl, dict_val)); log_write("metacall", LOG_LEVEL_DEBUG, "Discover object member %s, type %s", PyUnicode_AsUTF8(dict_key), - type_id_name(py_loader_impl_capi_to_value_type(dict_val))); + type_id_name(py_loader_impl_capi_to_value_type(impl, dict_val))); } } @@ -2415,6 +2783,12 @@ int py_loader_impl_destroy(loader_impl impl) Py_DECREF(py_impl->builtins_module); + Py_DECREF(py_impl->asyncio_module); + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_create_task); + Py_DECREF(py_impl->asyncio_get_event_loop); + Py_DECREF(py_impl->traceback_format_exception); Py_DECREF(py_impl->traceback_module); diff --git a/source/loaders/py_loader/source/py_loader_port.c b/source/loaders/py_loader/source/py_loader_port.c index 69045e151..595d46f34 100644 --- a/source/loaders/py_loader/source/py_loader_port.c +++ b/source/loaders/py_loader/source/py_loader_port.c @@ -355,7 +355,7 @@ static PyObject * py_loader_port_invoke(PyObject * self, PyObject * var_args) { PyObject * element = PyTuple_GetItem(var_args, args_count + 1); - value_args[args_count] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(element)); + value_args[args_count] = py_loader_impl_capi_to_value(impl, element, py_loader_impl_capi_to_value_type(impl, element)); } } diff --git a/source/tests/CMakeLists.txt b/source/tests/CMakeLists.txt index 65bafe1b6..1f9c97bde 100644 --- a/source/tests/CMakeLists.txt +++ b/source/tests/CMakeLists.txt @@ -150,6 +150,7 @@ add_subdirectory(metacall_python_pointer_test) add_subdirectory(metacall_python_reentrant_test) add_subdirectory(metacall_python_varargs_test) add_subdirectory(metacall_python_port_test) +add_subdirectory(metacall_python_async_test) add_subdirectory(metacall_map_test) add_subdirectory(metacall_map_await_test) add_subdirectory(metacall_initialize_test) diff --git a/source/tests/metacall_python_async_test/CMakeLists.txt b/source/tests/metacall_python_async_test/CMakeLists.txt new file mode 100644 index 000000000..3fc58fd40 --- /dev/null +++ b/source/tests/metacall_python_async_test/CMakeLists.txt @@ -0,0 +1,153 @@ +# Check if this loader is enabled +if(NOT OPTION_BUILD_LOADERS OR NOT OPTION_BUILD_LOADERS_PY OR NOT OPTION_BUILD_SCRIPTS OR NOT OPTION_BUILD_SCRIPTS_PY) + return() +endif() + +# +# Executable name and options +# + +# Target name +set(target metacall-python-async-test) +message(STATUS "Test ${target}") + +# +# Compiler warnings +# + +include(Warnings) + +# +# Compiler security +# + +include(SecurityFlags) + +# +# Sources +# + +set(include_path "${CMAKE_CURRENT_SOURCE_DIR}/include/${target}") +set(source_path "${CMAKE_CURRENT_SOURCE_DIR}/source") + +set(sources + ${source_path}/main.cpp + ${source_path}/metacall_python_async_test.cpp +) + +# Group source files +set(header_group "Header Files (API)") +set(source_group "Source Files") +source_group_by_path(${include_path} "\\\\.h$|\\\\.hpp$" + ${header_group} ${headers}) +source_group_by_path(${source_path} "\\\\.cpp$|\\\\.c$|\\\\.h$|\\\\.hpp$" + ${source_group} ${sources}) + +# +# Create executable +# + +# Build executable +add_executable(${target} + ${sources} +) + +# Create namespaced alias +add_executable(${META_PROJECT_NAME}::${target} ALIAS ${target}) + +# +# Project options +# + +set_target_properties(${target} + PROPERTIES + ${DEFAULT_PROJECT_OPTIONS} + FOLDER "${IDE_FOLDER}" +) + +# +# Include directories +# + +target_include_directories(${target} + PRIVATE + ${DEFAULT_INCLUDE_DIRECTORIES} + ${PROJECT_BINARY_DIR}/source/include +) + +# +# Libraries +# + +target_link_libraries(${target} + PRIVATE + ${DEFAULT_LIBRARIES} + + GTest + + ${META_PROJECT_NAME}::version + ${META_PROJECT_NAME}::preprocessor + ${META_PROJECT_NAME}::environment + ${META_PROJECT_NAME}::format + ${META_PROJECT_NAME}::log + ${META_PROJECT_NAME}::memory + ${META_PROJECT_NAME}::portability + ${META_PROJECT_NAME}::adt + ${META_PROJECT_NAME}::reflect + ${META_PROJECT_NAME}::dynlink + ${META_PROJECT_NAME}::detour + ${META_PROJECT_NAME}::serial + ${META_PROJECT_NAME}::configuration + ${META_PROJECT_NAME}::loader + ${META_PROJECT_NAME}::metacall +) + +# +# Compile definitions +# + +target_compile_definitions(${target} + PRIVATE + ${DEFAULT_COMPILE_DEFINITIONS} +) + +# +# Compile options +# + +target_compile_options(${target} + PRIVATE + ${DEFAULT_COMPILE_OPTIONS} +) + +# +# Linker options +# + +target_link_libraries(${target} + PRIVATE + ${DEFAULT_LINKER_OPTIONS} +) + +# +# Define test +# + +add_test(NAME ${target} + COMMAND $ +) + +# +# Define test properties +# + +set_property(TEST ${target} + PROPERTY LABELS ${target} MEMCHECK_IGNORE +) + +include(TestEnvironmentVariables) + +test_environment_variables(${target} + "" + ${TESTS_ENVIRONMENT_VARIABLES} +) diff --git a/source/tests/metacall_python_async_test/source/main.cpp b/source/tests/metacall_python_async_test/source/main.cpp new file mode 100644 index 000000000..14fb34603 --- /dev/null +++ b/source/tests/metacall_python_async_test/source/main.cpp @@ -0,0 +1,28 @@ +/* + * MetaCall Library by Parra Studios + * A library for providing a foreign function interface calls. + * + * Copyright (C) 2016 - 2020 Vicente Eduardo Ferrer Garcia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +int main(int argc, char * argv[]) +{ + ::testing::InitGoogleMock(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp b/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp new file mode 100644 index 000000000..06888881b --- /dev/null +++ b/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp @@ -0,0 +1,149 @@ +/* + * MetaCall Library by Parra Studios + * A library for providing a foreign function interface calls. + * + * Copyright (C) 2016 - 2020 Vicente Eduardo Ferrer Garcia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include +#include + +#include +#include + +class metacall_python_async_test : public testing::Test +{ +public: +}; + +TEST_F(metacall_python_async_test, DefaultConstructor) +{ + metacall_print_info(); + + ASSERT_EQ((int) 0, (int) metacall_initialize()); + + /* Python */ + #if defined(OPTION_BUILD_LOADERS_PY) + { + /* TODO: Create another test using Curio library? */ + const char buffer[] = + "import asyncio\n" + "async def my_sleep(n):\n" + "\tloop = asyncio.get_event_loop()\n" + "\tfuture = loop.create_future()\n" + "\tloop.call_later(n, future.set_result, None)\n" + "\tprint('before awaiting')\n" + "\tawait future\n" + "\tprint('after awaiting')\n" + "\treturn 'goodbye'\n"; + + EXPECT_EQ((int) 0, (int) metacall_load_from_memory("py", buffer, sizeof(buffer), NULL)); + + void * args[] = + { + metacall_value_create_long(2L) + }; + + struct await_data_type + { + std::mutex m; + std::condition_variable c; + } await_data; + + std::unique_lock lock(await_data.m); + + /* Test resolve */ + void * future = metacall_await("my_sleep", args, [](void * result, void * data) -> void * { + printf("Resolve C Callbackasdasdasd\n"); + struct await_data_type * await_data = static_cast(data); + //std::unique_lock lock(await_data->m); + + EXPECT_NE((void *) NULL, (void *) result); + + EXPECT_EQ((enum metacall_value_id) metacall_value_id(result), (enum metacall_value_id) METACALL_STRING); + + //EXPECT_EQ((long) 10, (long) metacall_value_to_long(result)); + + printf("Resolve C Callback\n"); + + await_data->c.notify_all(); + + return NULL; + }, [](void *, void * data) -> void * { + struct await_data_type * await_data = static_cast(data); + std::unique_lock lock(await_data->m); + + /* If we reach here, there's a serious bug in the C code that is called by python after the task is done */ + int never_executed = 0; + EXPECT_EQ((int) 1, (int) never_executed); + + printf("Reject C Callback\n"); + + await_data->c.notify_one(); + + return NULL; + }, static_cast(&await_data)); + + await_data.c.wait(lock); + + EXPECT_NE((void *) NULL, (void *) future); + + EXPECT_EQ((enum metacall_value_id) metacall_value_id(future), (enum metacall_value_id) METACALL_FUTURE); + + metacall_value_destroy(future); +#if 0 + /* Test reject */ + future = metacall_await("my_sleep_reject", args, [](void *, void *) -> void * { + int this_should_never_be_executed = 0; + + EXPECT_EQ((int) 1, (int) this_should_never_be_executed); + + printf("Resolve C Callback\n"); + + return NULL; + }, [](void * result, void * data) -> void * { + EXPECT_NE((void *) NULL, (void *) result); + + EXPECT_EQ((enum metacall_value_id) metacall_value_id(result), (enum metacall_value_id) METACALL_LONG); + + EXPECT_EQ((long) 10, (long) metacall_value_to_long(result)); + + EXPECT_NE((void *) NULL, (void *) data); + + struct async_context * ctx = static_cast(data); + + EXPECT_EQ((int) 234, (int) ctx->value); + + printf("Reject C Callback\n"); + + return metacall_value_create_double(15.0); + }, static_cast(&ctx)); + + EXPECT_NE((void *) NULL, (void *) future); + + EXPECT_EQ((enum metacall_value_id) metacall_value_id(future), (enum metacall_value_id) METACALL_FUTURE); + + metacall_value_destroy(future); +#endif + metacall_value_destroy(args[0]); + } + #endif /* OPTION_BUILD_LOADERS_PY */ + + EXPECT_EQ((int) 0, (int) metacall_destroy()); +} From 70fdc83abbc3c844f12cf35932dcc64c430c36ce Mon Sep 17 00:00:00 2001 From: Gil Arasa Verge Date: Thu, 17 Dec 2020 12:45:48 -0500 Subject: [PATCH 2/2] Try to reimplement with event loop in a background thread... not working --- .../loaders/py_loader/source/py_loader_impl.c | 292 +++++++++++++----- .../source/metacall_python_async_test.cpp | 65 ++-- 2 files changed, 249 insertions(+), 108 deletions(-) diff --git a/source/loaders/py_loader/source/py_loader_impl.c b/source/loaders/py_loader/source/py_loader_impl.c index 9e8ff3ea3..3939232f4 100644 --- a/source/loaders/py_loader/source/py_loader_impl.c +++ b/source/loaders/py_loader/source/py_loader_impl.c @@ -93,8 +93,13 @@ struct loader_impl_py_type PyObject *asyncio_module; PyObject *asyncio_isfuture; PyObject *asyncio_iscoroutinefunction; - PyObject *asyncio_create_task; - PyObject *asyncio_get_event_loop; + PyObject *asyncio_run_coroutine_threadsafe; + PyObject *asyncio_wrap_future; + PyObject *asyncio_loop; + + PyObject *py_task_callback_handler; + + PyObject *threading_module; #if (!defined(NDEBUG) || defined(DEBUG) || defined(_DEBUG) || defined(__DEBUG) || defined(__DEBUG__)) PyObject *gc_module; @@ -1284,6 +1289,8 @@ function_return function_py_interface_invoke(function func, function_impl impl, } } + /* TODO RUN USING run_in_executor */ + result = PyObject_CallObject(py_func->func, tuple_args); /* End of recursive call */ @@ -1389,7 +1396,7 @@ function_return function_py_interface_await(function func, function_impl impl, f const size_t args_size = size; const size_t signature_args_size = signature_count(s); type ret_type = signature_get_return(s); - PyObject *pytask = NULL; + PyObject *pyfuture = NULL; size_t args_count; loader_impl_py py_impl = loader_impl_get(py_func->impl); PyGILState_STATE gstate = PyGILState_Ensure(); @@ -1431,8 +1438,7 @@ function_return function_py_interface_await(function func, function_impl impl, f return NULL; } - /* TODO. should call https://docs.python.org/3.6/library/asyncio-eventloop.html#asyncio.AbstractEventLoop.close ??*/ - PyObject *loop = PyObject_CallFunctionObjArgs(py_impl->asyncio_get_event_loop, NULL); + pyfuture = PyObject_CallFunctionObjArgs(py_impl->asyncio_run_coroutine_threadsafe, coroutine, py_impl->asyncio_loop, NULL); if (PyErr_Occurred() != NULL) { py_loader_impl_error_print(py_impl); @@ -1440,37 +1446,13 @@ function_return function_py_interface_await(function func, function_impl impl, f return NULL; } - pytask = PyObject_CallFunctionObjArgs(py_impl->asyncio_create_task, coroutine, loop, NULL); - - loader_impl_py_await_invoke_callback_state callback_state = malloc(sizeof(struct loader_impl_py_await_invoke_callback_state_type)); - callback_state->resolve_callback = resolve_callback; - callback_state->reject_callback = reject_callback; - callback_state->py_impl = py_impl; - callback_state->context = context; - - /* Use this new Task member attribute to handle the callback pointers to the python task callback */ - PyObject * callback_status = PyCapsule_New(callback_state, NULL, NULL); - if (callback_status != NULL && PyObject_SetAttrString(pytask, "__metacall__capsule", callback_status)) - { - PyGILState_Release(gstate); - return NULL; - } - - /* Wrap the C callback around a PyObject so that the interpreter knows how to call back as if it was python code */ - PyMethodDef py_task_callback_handler_def = { - "metacall_task_callback_handler", - task_callback_handler_impl, - METH_O, - "MetaCall async invoke callback"}; - PyObject *py_task_callback_handler = PyCFunction_NewEx(&py_task_callback_handler_def, NULL, NULL); - /* When the future is done, automatically call the callback */ PyObject *add_done_callback_str = PyUnicode_FromString("add_done_callback"); - PyObject_CallMethodObjArgs(pytask, add_done_callback_str, py_task_callback_handler, NULL); + PyObject_CallMethodObjArgs(pyfuture, add_done_callback_str, py_impl->py_task_callback_handler, NULL); Py_DECREF(add_done_callback_str); - // TODO: ? Py_DECREF(py_task_callback_handler); - +#if 0 + pyfuture = PyObject_CallFunctionObjArgs(py_impl->asyncio_wrap_future, pyfuture, NULL); if (PyErr_Occurred() != NULL) { py_loader_impl_error_print(py_impl); @@ -1478,22 +1460,19 @@ function_return function_py_interface_await(function func, function_impl impl, f return NULL; } - PyObject *run_forever_str = PyUnicode_FromString("run_forever"); - PyObject_CallMethodObjArgs(loop, run_forever_str, NULL); - Py_DECREF(run_forever_str); - - if (PyErr_Occurred() != NULL) + loader_impl_py_await_invoke_callback_state callback_state = malloc(sizeof(struct loader_impl_py_await_invoke_callback_state_type)); + callback_state->resolve_callback = resolve_callback; + callback_state->reject_callback = reject_callback; + callback_state->py_impl = py_impl; + callback_state->context = context; + /* Use this new Task member attribute to handle the callback pointers to the python task callback */ + PyObject * callback_status = PyCapsule_New(callback_state, NULL, NULL); + if (callback_status != NULL && PyObject_SetAttrString(pyfuture, "__metacall__capsule", callback_status)) { - py_loader_impl_error_print(py_impl); PyGILState_Release(gstate); return NULL; } - - /* Loop will stop after an iteration. Perhaps this is not correct and something like asyncio.run_until_complete */ - /* should be used instead although removing the last line of that function which calls to future.result() */ - PyObject *stop_str = PyUnicode_FromString("stop"); - PyObject_CallMethodObjArgs(loop, stop_str, NULL); - Py_DECREF(stop_str); +#endif if (PyErr_Occurred() != NULL) { @@ -1502,7 +1481,6 @@ function_return function_py_interface_await(function func, function_impl impl, f return NULL; } - Py_DECREF(tuple_args); /* Variable arguments */ @@ -1511,7 +1489,7 @@ function_return function_py_interface_await(function func, function_impl impl, f free(values); } - if (pytask != NULL) + if (pyfuture != NULL) { value v = NULL; @@ -1519,16 +1497,16 @@ function_return function_py_interface_await(function func, function_impl impl, f if (ret_type == NULL) { - id = py_loader_impl_capi_to_value_type((loader_impl)py_impl, pytask); + id = py_loader_impl_capi_to_value_type((loader_impl)py_impl, pyfuture); } else { id = type_index(ret_type); } - v = py_loader_impl_capi_to_value(py_func->impl, pytask, id); + v = py_loader_impl_capi_to_value(py_func->impl, pyfuture, id); - Py_DECREF(pytask); + Py_DECREF(pyfuture); PyGILState_Release(gstate); @@ -1812,6 +1790,60 @@ int py_loader_impl_initialize_inspect(loader_impl impl, loader_impl_py py_impl) return 1; } +int py_loader_impl_initialize_threading_module(loader_impl impl, loader_impl_py py_impl) +{ + (void)impl; + + PyObject *module_name = PyUnicode_DecodeFSDefault("threading"); + + py_impl->threading_module = PyImport_Import(module_name); + + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + + Py_DECREF(module_name); + + return 1; + } + + if (py_impl->asyncio_module == NULL) + { + log_write("metacall", LOG_LEVEL_ERROR, "Error getting threading module"); + return 1; + } + + Py_DECREF(module_name); + + return 0; +} + +PyObject * py_background_thread_create_event_loop(PyObject * self, PyObject * py_impl_capsule) +{ + (void)self; + + loader_impl_py py_impl = PyCapsule_GetPointer(py_impl_capsule, NULL); + + PyObject * set_event_loop = PyObject_GetAttrString(py_impl->asyncio_module, "set_event_loop"); + PyObject_CallFunctionObjArgs(set_event_loop, py_impl->asyncio_loop, NULL); + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + Py_RETURN_NONE; + } + + PyObject *run_forever_str = PyUnicode_FromString("run_forever"); + PyObject_CallMethodObjArgs(py_impl->asyncio_loop, run_forever_str, NULL); + if (PyErr_Occurred() != NULL) + { + py_loader_impl_error_print(py_impl); + } + + Py_DECREF(run_forever_str); + + Py_RETURN_NONE; +} + int py_loader_impl_initialize_asyncio_module(loader_impl impl, loader_impl_py py_impl) { (void)impl; @@ -1828,50 +1860,131 @@ int py_loader_impl_initialize_asyncio_module(loader_impl impl, loader_impl_py py return 1; } + Py_DECREF(module_name); - if (py_impl->asyncio_module != NULL) + if (py_impl->asyncio_module == NULL || py_impl->threading_module == NULL) { - py_impl->asyncio_isfuture = PyObject_GetAttrString(py_impl->asyncio_module, "isfuture"); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio module or threading module not loaded"); + return 1; + } - if (py_impl->asyncio_isfuture == NULL) - { - log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.isfuture(obj)"); - return 1; - } + py_impl->asyncio_isfuture = PyObject_GetAttrString(py_impl->asyncio_module, "isfuture"); - py_impl->asyncio_iscoroutinefunction = PyObject_GetAttrString(py_impl->asyncio_module, "iscoroutinefunction"); + if (py_impl->asyncio_isfuture == NULL) + { + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.isfuture(obj)"); + return 1; + } - if (py_impl->asyncio_iscoroutinefunction == NULL) - { - Py_DECREF(py_impl->asyncio_isfuture); - log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.iscoroutinefunction(func)"); - return 1; - } + py_impl->asyncio_iscoroutinefunction = PyObject_GetAttrString(py_impl->asyncio_module, "iscoroutinefunction"); + + if (py_impl->asyncio_iscoroutinefunction == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.iscoroutinefunction(func)"); + return 1; + } - py_impl->asyncio_create_task = PyObject_GetAttrString(py_impl->asyncio_module, "run_coroutine_threadsafe"); + py_impl->asyncio_run_coroutine_threadsafe = PyObject_GetAttrString(py_impl->asyncio_module, "run_coroutine_threadsafe"); - if (py_impl->asyncio_create_task == NULL) - { - Py_DECREF(py_impl->asyncio_isfuture); - Py_DECREF(py_impl->asyncio_iscoroutinefunction); - log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.run_coroutine_threadsafe(coro, loop)"); - return 1; - } + if (py_impl->asyncio_run_coroutine_threadsafe == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.run_coroutine_threadsafe(coro, loop)"); + return 1; + } + + py_impl->asyncio_wrap_future = PyObject_GetAttrString(py_impl->asyncio_module, "wrap_future"); + if (py_impl->asyncio_wrap_future == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_run_coroutine_threadsafe); + + log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.wrap_future()"); + return 1; + } + + /* Wrap the C callback around a PyObject so that the interpreter knows how to call back as if it was python code */ + PyMethodDef py_task_callback_handler_def = { + "metacall_task_callback_handler", + task_callback_handler_impl, + METH_O, + "MetaCall async invoke callback"}; + py_impl->py_task_callback_handler = PyCFunction_NewEx(&py_task_callback_handler_def, NULL, NULL); + + + PyObject * new_event_loop_func = PyObject_GetAttrString(py_impl->asyncio_module, "new_event_loop"); + /* TODO. should call https://docs.python.org/3.6/library/asyncio-eventloop.html#asyncio.AbstractEventLoop.close WHEN DESTROYING */ + PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop_func, NULL); + if (PyErr_Occurred() != NULL || loop == NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_run_coroutine_threadsafe); + Py_DECREF(py_impl->asyncio_wrap_future); + py_loader_impl_error_print(py_impl); + return 1; + } + py_impl->asyncio_loop = loop; + Py_XDECREF(new_event_loop_func); + + PyMethodDef py_background_thread_create_event_loop_def = { + "metacall_background_thread_create_event_loop", + py_background_thread_create_event_loop, + METH_O, + "MetaCall asyncio loop background thread create"}; + PyObject *pyobj_background_thread_create_event_loop = PyCFunction_NewEx(&py_background_thread_create_event_loop_def, py_impl->asyncio_module, NULL); + + PyObject *thread_name_str = PyUnicode_FromString("Metacall asyncio event loop"); + PyObject *thread_object = PyObject_GetAttrString(py_impl->threading_module, "Thread"); + + PyObject *emptyArgs = PyTuple_New(0); + + PyObject * py_impl_capsule = PyCapsule_New(py_impl, NULL, NULL); - py_impl->asyncio_get_event_loop = PyObject_GetAttrString(py_impl->asyncio_module, "get_event_loop"); + PyObject * customFuncArgsTuple = PyTuple_New(1); + PyTuple_SET_ITEM(customFuncArgsTuple, 0, py_impl_capsule); - if (py_impl->asyncio_get_event_loop == NULL) - { - Py_DECREF(py_impl->asyncio_isfuture); - Py_DECREF(py_impl->asyncio_iscoroutinefunction); - Py_DECREF(py_impl->asyncio_create_task); - log_write("metacall", LOG_LEVEL_ERROR, "Error getting asyncio.get_event_loop()"); - return 1; - } + PyObject *kargs = PyDict_New(); + PyDict_SetItemString(kargs, "target", pyobj_background_thread_create_event_loop); + PyDict_SetItemString(kargs, "name", thread_name_str); + PyDict_SetItemString(kargs, "args", customFuncArgsTuple); + + PyObject * thread = PyObject_Call(thread_object, emptyArgs, kargs); + if (PyErr_Occurred() != NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_run_coroutine_threadsafe); + Py_DECREF(py_impl->asyncio_loop); + Py_DECREF(py_impl->asyncio_wrap_future); + py_loader_impl_error_print(py_impl); + return 1; } - Py_DECREF(module_name); + Py_DECREF(py_impl_capsule); + + PyObject *start_thread_str = PyUnicode_FromString("start"); + PyObject_CallMethodObjArgs(thread, start_thread_str, NULL); + if (PyErr_Occurred() != NULL) + { + Py_DECREF(py_impl->asyncio_isfuture); + Py_DECREF(py_impl->asyncio_iscoroutinefunction); + Py_DECREF(py_impl->asyncio_run_coroutine_threadsafe); + Py_DECREF(py_impl->asyncio_loop); + Py_DECREF(py_impl->asyncio_wrap_future); + py_loader_impl_error_print(py_impl); + return 1; + } + Py_DECREF(start_thread_str); + Py_DECREF(kargs); + Py_DECREF(customFuncArgsTuple); + Py_DECREF(emptyArgs); + Py_DECREF(thread_name_str); + return 0; } @@ -2030,6 +2143,15 @@ loader_impl_data py_loader_impl_initialize(loader_impl impl, configuration confi return NULL; } + if (py_loader_impl_initialize_threading_module(impl, py_impl) != 0) + { + PyGILState_Release(gstate); + + free(py_impl); + + return NULL; + } + if (py_loader_impl_initialize_asyncio_module(impl, py_impl) != 0) { PyGILState_Release(gstate); @@ -2786,8 +2908,12 @@ int py_loader_impl_destroy(loader_impl impl) Py_DECREF(py_impl->asyncio_module); Py_DECREF(py_impl->asyncio_isfuture); Py_DECREF(py_impl->asyncio_iscoroutinefunction); - Py_DECREF(py_impl->asyncio_create_task); - Py_DECREF(py_impl->asyncio_get_event_loop); + Py_DECREF(py_impl->asyncio_run_coroutine_threadsafe); + Py_DECREF(py_impl->asyncio_wrap_future); + Py_DECREF(py_impl->asyncio_loop); + Py_DECREF(py_impl->py_task_callback_handler); + + Py_DECREF(py_impl->threading_module); Py_DECREF(py_impl->traceback_format_exception); diff --git a/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp b/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp index 06888881b..3a65e2805 100644 --- a/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp +++ b/source/tests/metacall_python_async_test/source/metacall_python_async_test.cpp @@ -44,6 +44,9 @@ TEST_F(metacall_python_async_test, DefaultConstructor) /* TODO: Create another test using Curio library? */ const char buffer[] = "import asyncio\n" + + "print('sync message')\n" + "async def my_sleep(n):\n" "\tloop = asyncio.get_event_loop()\n" "\tfuture = loop.create_future()\n" @@ -51,6 +54,16 @@ TEST_F(metacall_python_async_test, DefaultConstructor) "\tprint('before awaiting')\n" "\tawait future\n" "\tprint('after awaiting')\n" + "\treturn 'goodbye'\n" + + "async def my_sleep_reject(n):\n" + "\tloop = asyncio.get_event_loop()\n" + "\tfuture = loop.create_future()\n" + "\tloop.call_later(n, future.set_result, None)\n" + "\tprint('before awaiting reject')\n" + "\tawait future\n" + "\tprint('after awaiting reject')\n" + "\tfuture.cancel()\n" "\treturn 'goodbye'\n"; EXPECT_EQ((int) 0, (int) metacall_load_from_memory("py", buffer, sizeof(buffer), NULL)); @@ -70,16 +83,14 @@ TEST_F(metacall_python_async_test, DefaultConstructor) /* Test resolve */ void * future = metacall_await("my_sleep", args, [](void * result, void * data) -> void * { - printf("Resolve C Callbackasdasdasd\n"); + printf("Got into C callback at least\n"); struct await_data_type * await_data = static_cast(data); - //std::unique_lock lock(await_data->m); + std::unique_lock lock(await_data->m); EXPECT_NE((void *) NULL, (void *) result); EXPECT_EQ((enum metacall_value_id) metacall_value_id(result), (enum metacall_value_id) METACALL_STRING); - //EXPECT_EQ((long) 10, (long) metacall_value_to_long(result)); - printf("Resolve C Callback\n"); await_data->c.notify_all(); @@ -100,47 +111,51 @@ TEST_F(metacall_python_async_test, DefaultConstructor) return NULL; }, static_cast(&await_data)); - await_data.c.wait(lock); - EXPECT_NE((void *) NULL, (void *) future); EXPECT_EQ((enum metacall_value_id) metacall_value_id(future), (enum metacall_value_id) METACALL_FUTURE); + await_data.c.wait(lock); + metacall_value_destroy(future); -#if 0 + /* Test reject */ - future = metacall_await("my_sleep_reject", args, [](void *, void *) -> void * { - int this_should_never_be_executed = 0; + future = metacall_await("my_sleep_reject", args, [](void * result, void * data) -> void * { + struct await_data_type * await_data = static_cast(data); + std::unique_lock lock(await_data->m); + + (void)result; - EXPECT_EQ((int) 1, (int) this_should_never_be_executed); + /* If we reach here, there's a serious bug in the C code that is called by python after the task is done */ + int never_executed = 0; + EXPECT_EQ((int) 1, (int) never_executed); printf("Resolve C Callback\n"); - return NULL; - }, [](void * result, void * data) -> void * { - EXPECT_NE((void *) NULL, (void *) result); - - EXPECT_EQ((enum metacall_value_id) metacall_value_id(result), (enum metacall_value_id) METACALL_LONG); - - EXPECT_EQ((long) 10, (long) metacall_value_to_long(result)); - - EXPECT_NE((void *) NULL, (void *) data); + await_data->c.notify_one(); - struct async_context * ctx = static_cast(data); + return NULL; + }, [](void *, void * data) -> void * { + struct await_data_type * await_data = static_cast(data); + std::unique_lock lock(await_data->m); + + printf("Reject C Callback\n"); - EXPECT_EQ((int) 234, (int) ctx->value); + await_data->c.notify_one(); - printf("Reject C Callback\n"); + return NULL; + }, static_cast(&await_data)); - return metacall_value_create_double(15.0); - }, static_cast(&ctx)); EXPECT_NE((void *) NULL, (void *) future); EXPECT_EQ((enum metacall_value_id) metacall_value_id(future), (enum metacall_value_id) METACALL_FUTURE); + await_data.c.wait(lock); + metacall_value_destroy(future); -#endif + + metacall_value_destroy(args[0]); } #endif /* OPTION_BUILD_LOADERS_PY */