From cf17654774a1ad18857c6c437b2ea07c16bb9170 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 12 Jan 2022 17:49:40 +0100 Subject: [PATCH 01/40] Removes use of deprecated libqi features. Change-Id: I46cf17e9cc68de45ba6a2e5000bc58858ddbcac1 Reviewed-on: http://gerrit2.aldebaran.lan/1764 Reviewed-by: philippe.martin Reviewed-by: jmonnon Tested-by: vincent.palancher --- src/pyasync.cpp | 7 +++++-- tests/test_property.cpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pyasync.cpp b/src/pyasync.cpp index b66fdd8..ba87b3a 100644 --- a/src/pyasync.cpp +++ b/src/pyasync.cpp @@ -102,7 +102,10 @@ void exportAsync(::py::module& m) "Set the callback used by the periodic task, this function can only be " "called once.\n" ":param callable: a python callable, could be a method or a function.")) - .def("setUsPeriod", &PeriodicTask::setUsPeriod, + .def("setUsPeriod", + [](PeriodicTask& task, qi::int64_t usPeriod) { + task.setPeriod(qi::MicroSeconds(usPeriod)); + }, call_guard(), "usPeriod"_a, doc("Set the call interval in microseconds.\n" "This call will wait until next callback invocation to apply the " @@ -112,7 +115,7 @@ void exportAsync(::py::module& m) ".. code-block:: python\n" "\n" " task.stop()\n" - " task.setUsPeriod()\n" + " task.setUsPeriod(100)\n" " task.start()\n" ":param usPeriod: the period in microseconds")) .def("start", &PeriodicTask::start, call_guard(), diff --git a/tests/test_property.cpp b/tests/test_property.cpp index 5687550..1b74acf 100644 --- a/tests/test_property.cpp +++ b/tests/test_property.cpp @@ -240,7 +240,7 @@ TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnectedAsync) auto success = qi::py::test::toFutOf( pyProp.attr("disconnect")(id, py::arg("_async") = true) - .cast()); + .cast()).value(); EXPECT_TRUE(success); } From cadda661108112d0edfe5bb0b38e14b43558e1a2 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Mon, 25 Oct 2021 12:09:04 +0200 Subject: [PATCH 02/40] misc: Fixes Python version check code. The previous code was checking if one of the tuple was less than the other, but the tuples contain 3 strings (e.g. `('3', '5', '0')`). Unfortunately, string comparison does not behave like we naively thought it did, as for instance, '5' is not less than '10'. This was causing a failure for the check for Python 3.10. This patch rewrites this check using the `packaging.version` module which implements correct version comparison according to PEP 440. Change-Id: Icacb8fb0a2c0cbaa0b159a9269a3fbde152ee412 Reviewed-on: http://gerrit2.aldebaran.lan/1613 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- pyproject.toml | 2 +- qi/__init__.py | 9 ++++++--- setup.py | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 390d211..5ad53cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] requires = ["setuptools>=47.1", "wheel>=0.34", "scikit-build>=0.10.1", - "cmake>=3.17", "ninja>=1.5"] + "cmake>=3.17", "ninja>=1.5", "packaging~=21"] diff --git a/qi/__init__.py b/qi/__init__.py index 9595052..7f5f83e 100644 --- a/qi/__init__.py +++ b/qi/__init__.py @@ -6,9 +6,12 @@ """LibQi Python bindings.""" import platform -py_version = platform.python_version_tuple() -if py_version < ('3', '5'): - raise RuntimeError('Python version 3.5+ is required.') +from packaging import version + +py_version = version.parse(platform.python_version()) +min_version = version.parse('3.5') +if py_version < min_version: + raise RuntimeError('Python 3.5+ is required.') import sys # noqa: E402 import atexit # noqa: E402 diff --git a/setup.py b/setup.py index 5718b87..ac29f47 100755 --- a/setup.py +++ b/setup.py @@ -5,10 +5,12 @@ import platform from setuptools import find_packages from skbuild import setup +from packaging import version -py_version = platform.python_version_tuple() -if py_version < ('3', '5'): - raise RuntimeError('Python 3.5+ is required.') +py_version = version.parse(platform.python_version()) +min_version = version.parse('3.5') +if py_version < min_version: + raise RuntimeError('Python 3.5+ is required.') here = os.path.abspath(os.path.dirname(__file__)) From 07e0047b644b95fa339a29e040f665acdafc5aea Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 19 Jan 2022 16:52:55 +0100 Subject: [PATCH 03/40] setup: Adds runtime dependencies. This patch adds runtime dependencies information in `setup.py` so it appears in the metadata of the resulting wheel package and may be recognized by `pip`. It also adds details about build dependencies. Change-Id: I04eae9d5b5d9d5fc56a07bb2d6e3ec80b9270ecc Reviewed-on: http://gerrit2.aldebaran.lan/1793 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- pyproject.toml | 18 ++++++++++++++++-- setup.py | 7 +++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ad53cd..cc4c529 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,17 @@ +[project] +dependencies = [ + "packaging" +] + +# PEP 518 [build-system] -requires = ["setuptools>=47.1", "wheel>=0.34", "scikit-build>=0.10.1", - "cmake>=3.17", "ninja>=1.5", "packaging~=21"] +# Minimum requirements for the build system to execute. +requires = [ + "setuptools >= 47, < 51", # setuptools 51 dropped support for Python 3.5. + "wheel >= 0.34", # tested with 0.34. + "scikit-build >= 0.10", # tested with 0.10. + "cmake ~= 3.16", # CMake >= 3.16, CMake < 4. + "ninja ~= 1", # ninja >= 1, ninja < 2. + "toml ~= 10", + "packaging ~= 21", +] diff --git a/setup.py b/setup.py index ac29f47..66c1e53 100755 --- a/setup.py +++ b/setup.py @@ -3,15 +3,21 @@ import os import platform +import pathlib from setuptools import find_packages from skbuild import setup from packaging import version +import toml py_version = version.parse(platform.python_version()) min_version = version.parse('3.5') if py_version < min_version: raise RuntimeError('Python 3.5+ is required.') +# Parse `pyproject.toml` runtime dependencies. +pyproject_data = toml.loads(pathlib.Path('pyproject.toml').read_text()) +pyproject_deps = pyproject_data['project']['dependencies'] + here = os.path.abspath(os.path.dirname(__file__)) version = {} @@ -36,6 +42,7 @@ author_email='release@softbankrobotics.com', license='BSD 3-Clause License', python_requires='~=3.5', + install_requires=pyproject_deps, packages=find_packages(exclude=[ '*.test', '*.test.*', 'test.*', 'test' ]), From 2ae3f9335138345e5599fbb79310ae507fcda779 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 26 Oct 2021 18:43:13 +0200 Subject: [PATCH 04/40] cmake.install: Uses `patchelf` when available to set the runtime path of external shared libraries. This patch uses the `patchelf` tool to set the runtime path of external shared libraries wrapped in the package to '$ORIGIN' or '@loader_path'. These special values refer to the path of the shared library itself, depending on the system. All the shared libraries that are wrapped in the package are extracted in the same directory when the package is installed. This allows us to ensure that any shared library that gets wrapped in the package can find its dependencies also wrapped in the package. Change-Id: I5e1ec841942e20a7dcefa5d90a67073635709272 Reviewed-on: http://gerrit2.aldebaran.lan/1622 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- cmake/install_runtime_dependencies.cmake | 27 ++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cmake/install_runtime_dependencies.cmake b/cmake/install_runtime_dependencies.cmake index 648e9d3..ae6f787 100644 --- a/cmake/install_runtime_dependencies.cmake +++ b/cmake/install_runtime_dependencies.cmake @@ -58,6 +58,21 @@ if(_unresolved_runtime_dependencies) message(STATUS "Some dependencies of qi_python and qi are unresolved: ${_unresolved_runtime_dependencies}") endif() +if(UNIX OR APPLE) + find_program(_patchelf "patchelf") + if(_patchelf) + if(APPLE) + set(_patchelf_runtime_path "@loader_path") + elseif(UNIX) + set(_patchelf_runtime_path "$ORIGIN") + endif() + else() + message(WARNING "The target system seems to be using ELF binaries, but the `patchelf` \ +tool could not be found. The installed runtime dependencies might not have their runtime path \ +set correctly to run. You might want to install `patchelf` on your system.") + endif() +endif() + foreach(_needed_dep IN LISTS _runtime_dependencies) set(_dep "${_needed_dep}") @@ -80,11 +95,19 @@ foreach(_needed_dep IN LISTS _runtime_dependencies) endwhile() file(INSTALL "${_dep}" DESTINATION "${_module_install_dir}") + get_filename_component(_dep_filename "${_dep}" NAME) + set(_installed_dep "${_module_install_dir}/${_dep_filename}") + + if(_patchelf) + message(STATUS "Set runtime path of runtime dependency \"${_installed_dep}\" to \"${_patchelf_runtime_path}\"") + file(TO_NATIVE_PATH "${_installed_dep}" _native_installed_dep) + execute_process(COMMAND "${_patchelf}" + # Sets the RPATH/RUNPATH or may replace the RPATH by a RUNPATH, depending on the platform. + --set-rpath "${_patchelf_runtime_path}" "${_native_installed_dep}") + endif() if(NOT "${_dep}" STREQUAL "${_needed_dep}") - get_filename_component(_dep_filename "${_dep}" NAME) get_filename_component(_needed_dep_filename "${_needed_dep}" NAME) - set(_installed_dep "${_module_install_dir}/${_dep_filename}") set(_installed_needed_dep "${_module_install_dir}/${_needed_dep_filename}") message(STATUS "Renaming ${_installed_dep} into ${_installed_needed_dep}") file(RENAME "${_installed_dep}" "${_installed_needed_dep}") From da9074f3802bc3dec36a14b179420d37f66b9250 Mon Sep 17 00:00:00 2001 From: Justine LANCA Date: Thu, 13 Jan 2022 18:48:15 +0100 Subject: [PATCH 05/40] tests: Adds a test exposing a leak for objects with a property or a signal. Objects with a qi::Property or a qi::Signal don't seem to be destroyed when they should. This patch adds a test that reproduces this problem. Change-Id: Ieab21206e7a553025a2e6547d788792462421878 Reviewed-on: http://gerrit2.aldebaran.lan/1783 Reviewed-by: philippe.martin Reviewed-by: jmonnon Tested-by: vincent.palancher --- qi/test/test_module.py | 31 ++++++++++++ tests/moduletest.cpp | 112 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 140 insertions(+), 3 deletions(-) diff --git a/qi/test/test_module.py b/qi/test/test_module.py index eb41e39..8796116 100644 --- a/qi/test/test_module.py +++ b/qi/test/test_module.py @@ -41,3 +41,34 @@ def test_module_service(): cat = session.service("Cat") assert cat.meow(3) == 'meow' + + +def test_module_service_object_lifetime(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + cat = session.service("Cat") + + # We use `del` to release the reference to an object, but there is no + # guarantee that the interpreter will finalize the object right away. + # In CPython, this is "mostly" guaranteed as long as it is the last + # reference to the object and that there is no cyclic dependency of + # reference anywhere that might hold the reference. + # As there is no practical alternative, we consider that assuming the + # object is finalized immediately is an acceptable hypothesis. + + # Purr has a property, Play has a signal, Sleep has none. + sleep = cat.makeSleep() + assert cat.nbSleep() == 1 + del sleep + assert cat.nbSleep() == 0 + + purr = cat.makePurr() + assert cat.nbPurr() == 1 + del purr + assert cat.nbPurr() == 0 + + play = cat.makePlay() + assert cat.nbPlay() == 1 + del play + assert cat.nbPlay() == 0 diff --git a/tests/moduletest.cpp b/tests/moduletest.cpp index 023ea24..b9e3108 100644 --- a/tests/moduletest.cpp +++ b/tests/moduletest.cpp @@ -20,6 +20,77 @@ int Mouse::squeak() return 18; } +class Purr +{ +public: + Purr(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " purr constructor "; + ++(*counter); + } + ~Purr() + { + qiLogInfo() << this << " purr destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " purring"; + } + qi::Property volume; +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Purr, run, volume); + +class Sleep +{ +public: + Sleep(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " sleep constructor "; + ++(*counter); + } + ~Sleep() + { + qiLogInfo() << this << " sleep destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " sleeping"; + } +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Sleep, run); + +class Play +{ +public: + Play(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " play constructor "; + ++(*counter); + } + ~Play() + { + qiLogInfo() << this << " play destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " playing"; + } + qi::Signal caught; +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Play, run, caught); + class Cat { public: @@ -35,24 +106,59 @@ class Cat return boost::make_shared(); } + boost::shared_ptr makePurr() const + { + return boost::make_shared(purrCounter); + } + + boost::shared_ptr makeSleep() const + { + return boost::make_shared(sleepCounter); + } + + boost::shared_ptr makePlay() const + { + return boost::make_shared(playCounter); + } + + int nbPurr() + { + return purrCounter->load(); + } + + int nbSleep() + { + return sleepCounter->load(); + } + + int nbPlay() + { + return playCounter->load(); + } + qi::Property hunger; qi::Property boredom; qi::Property cuteness; + + std::shared_ptr> purrCounter = std::make_shared>(0); + std::shared_ptr> sleepCounter = std::make_shared>(0); + std::shared_ptr> playCounter = std::make_shared>(0); }; -QI_REGISTER_OBJECT(Cat, meow, cloneMe, hunger, boredom, cuteness); +QI_REGISTER_OBJECT(Cat, meow, cloneMe, hunger, boredom, cuteness, + makePurr, makeSleep, makePlay, nbPurr, nbSleep, nbPlay); Cat::Cat() { qiLogInfo() << "Cat constructor"; } -Cat::Cat(const std::string& s) +Cat::Cat(const std::string& s) : Cat() { qiLogInfo() << "Cat string constructor: " << s; } -Cat::Cat(const qi::SessionPtr& s) +Cat::Cat(const qi::SessionPtr& s) : Cat() { qiLogInfo() << "Cat string constructor with session"; s->services(); // SEGV? From 84f789273f2d46a7ae28dd321c0deb0aa4997f39 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 19 Jan 2022 13:48:02 +0100 Subject: [PATCH 06/40] qipython.guard: Adds GIL manipulation types. In `pybind11`, GIL manipulation RAII guard types are not reentrant (at least `gil_scoped_release` is not) which means we cannot reliably use it in subfunctions without having previous knowledge of the GIL state. This patch adds new re-entrant types which wrap `pybind11` own guards. These new types also check at destruction if the interpreter is finalizing, potentially (if `pybind11` supports it) disabling the guard, thus avoiding a thread termination. It also replaces any occurrences of previous guards with the new ones. Change-Id: Ie9b134ce05df1787ae5c7b93eae7e750fb317123 Reviewed-on: http://gerrit2.aldebaran.lan/1792 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- qipython/common.hpp | 55 +++++++++++--------- qipython/pyfuture.hpp | 4 +- qipython/pyguard.hpp | 110 +++++++++++++++++++++++++++++++++++++++- src/pyapplication.cpp | 21 ++++---- src/pyasync.cpp | 19 +++---- src/pyclock.cpp | 3 +- src/pyexport.cpp | 3 +- src/pyfuture.cpp | 48 +++++++++--------- src/pylog.cpp | 11 ++-- src/pymodule.cpp | 6 +-- src/pyobject.cpp | 40 +++++++-------- src/pypath.cpp | 41 +++++++-------- src/pyproperty.cpp | 35 ++++++------- src/pysession.cpp | 9 ++-- src/pysignal.cpp | 43 ++++++++-------- src/pystrand.cpp | 15 +++--- src/pytranslator.cpp | 13 ++--- src/pytypes.cpp | 85 ++++++++++++++++--------------- tests/common.hpp | 9 ++-- tests/test_guard.cpp | 18 +++++++ tests/test_module.cpp | 5 +- tests/test_object.cpp | 34 ++++++------- tests/test_property.cpp | 37 +++++++------- tests/test_qipython.cpp | 3 +- tests/test_signal.cpp | 15 +++--- tests/test_types.cpp | 15 +++--- 26 files changed, 422 insertions(+), 275 deletions(-) diff --git a/qipython/common.hpp b/qipython/common.hpp index f9b99e2..55fbf78 100644 --- a/qipython/common.hpp +++ b/qipython/common.hpp @@ -19,6 +19,23 @@ #include #include + +// MAJOR, MINOR and PATCH must be in [0, 255]. +#define QI_PYBIND11_VERSION(MAJOR, MINOR, PATCH) \ + (((MAJOR) << 16) | \ + ((MINOR) << 8) | \ + ((PATCH) << 0)) + +#ifdef PYBIND11_VERSION_HEX +// Remove the lowest 8 bits which represent the serial and level version components. +# define QI_CURRENT_PYBIND11_VERSION (PYBIND11_VERSION_HEX >> 8) +#else +# define QI_CURRENT_PYBIND11_VERSION \ + QI_PYBIND11_VERSION(PYBIND11_VERSION_MAJOR, \ + PYBIND11_VERSION_MINOR, \ + PYBIND11_VERSION_PATCH) +#endif + namespace qi { namespace py @@ -93,32 +110,20 @@ boost::optional extractKeywordArg(pybind11::dict kwargs, return pyArg.cast(); } -/// A deleter that deletes the pointer outside of the GIL. -/// -/// Useful for types that might deadlock on destruction if they keep the GIL -/// locked. -struct DeleteOutsideGIL -{ - template - void operator()(T* ptr) const - { - pybind11::gil_scoped_release unlock; - delete ptr; - } -}; - -/// Delay the destruction of an object to a thread. -struct DeleteInOtherThread +/// Returns whether or not the interpreter is finalizing. If the information +/// could not be fetched, the function returns an empty optional. Otherwise, it +/// returns an optional set with a boolean stating if the interpreter is indeed +/// finalizing or not. +inline boost::optional interpreterIsFinalizing() { - template - void operator()(T* ptr) const - { - pybind11::gil_scoped_release unlock; - auto fut = std::async(std::launch::async, [](std::unique_ptr) {}, - std::unique_ptr(ptr)); - QI_IGNORE_UNUSED(fut); - } -}; +// `_Py_IsFinalizing` is only available on CPython 3.7+ +#if PY_VERSION_HEX >= 0x03070000 + return boost::make_optional(_Py_IsFinalizing() != 0); +#else + // There is no way of knowing on older versions. + return {}; +#endif +} } // namespace py } // namespace qi diff --git a/qipython/pyfuture.hpp b/qipython/pyfuture.hpp index 85b810f..82468e2 100644 --- a/qipython/pyfuture.hpp +++ b/qipython/pyfuture.hpp @@ -25,12 +25,12 @@ using Promise = qi::Promise; // future or its value. inline pybind11::object resultObject(const Future& fut, bool async) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; if (async) return castToPyObject(fut); // Wait for the future outside of the GIL. - auto res = invokeGuarded(qi::SrcFuture{}, fut); + auto res = invokeGuarded(qi::SrcFuture{}, fut); return castToPyObject(res); } diff --git a/qipython/pyguard.hpp b/qipython/pyguard.hpp index 2bc42c4..f09fcd6 100644 --- a/qipython/pyguard.hpp +++ b/qipython/pyguard.hpp @@ -17,6 +17,12 @@ namespace qi namespace py { +/// Returns whether or not the GIL is currently held by the current thread. +inline bool currentThreadHoldsGil() +{ + return PyGILState_Check() == 1; +} + /// DefaultConstructible Guard /// Procedure<_ (Args)> F template @@ -121,13 +127,115 @@ class Guarded explicit operator const Object&() const { return object(); } }; +/// G == pybind11::gil_scoped_acquire || G == pybind11::gil_scoped_release +template +void pybind11GuardDisarm(G& guard) +{ + QI_IGNORE_UNUSED(guard); +// The disarm API was introduced in v2.6.2. +#if QI_CURRENT_PYBIND11_VERSION >= QI_PYBIND11_VERSION(2,6,2) + guard.disarm(); +#endif +} + } // namespace detail +/// RAII utility type that guarantees that the GIL is locked for the scope of +/// the lifetime of the object. +/// +/// This type is re-entrant. +/// +/// postcondition: `GILAcquire acq;` establishes `currentThreadHoldsGil()` +struct GILAcquire +{ + inline GILAcquire() + { + // `gil_scoped_acquire` is re-entrant by itself, so we don't need to check + // whether or not the GIL is already held by the current thread. + QI_ASSERT(currentThreadHoldsGil()); + } + + GILAcquire(const GILAcquire&) = delete; + GILAcquire& operator=(const GILAcquire&) = delete; + + inline ~GILAcquire() + { + const auto optIsFinalizing = interpreterIsFinalizing(); + const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; + if (definitelyFinalizing) + detail::pybind11GuardDisarm(_acq); + } + +private: + pybind11::gil_scoped_acquire _acq; +}; + +/// RAII utility type that guarantees that the GIL is unlocked for the scope of +/// the lifetime of the object. +/// +/// This type is re-entrant. +/// +/// postcondition: `GILRelease rel;` establishes `!currentThreadHoldsGil()` +struct GILRelease +{ + inline GILRelease() + { + if (currentThreadHoldsGil()) + _release.emplace(); + QI_ASSERT(!currentThreadHoldsGil()); + } + + GILRelease(const GILRelease&) = delete; + GILRelease& operator=(const GILRelease&) = delete; + + inline ~GILRelease() + { + const auto optIsFinalizing = interpreterIsFinalizing(); + const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; + if (_release && definitelyFinalizing) + detail::pybind11GuardDisarm(*_release); + } + +private: + boost::optional _release; +}; + /// Wraps a pybind11::object value and locks the GIL on copy and destruction. /// /// This is useful for instance to put pybind11 objects in lambda functions so /// that they can be copied around safely. -using GILGuardedObject = detail::Guarded; +using GILGuardedObject = detail::Guarded; + +/// Deleter that deletes the pointer outside the GIL. +/// +/// Useful for types that might deadlock on destruction if they keep the GIL +/// locked. +struct DeleteOutsideGIL +{ + template + void operator()(T* ptr) const + { + GILRelease unlock; + delete ptr; + } +}; + +/// Deleter that delays the destruction of an object to another thread. +struct DeleteInOtherThread +{ + template + void operator()(T* ptr) const + { + GILRelease unlock; + // `std::async` returns an object, that unless moved from will block when + // destroyed until the task is complete, which is unwanted behavior here. + // Therefore we just take the result of the `std::async` call into a local + // variable and then ignore it. + auto fut = std::async(std::launch::async, [](std::unique_ptr) {}, + std::unique_ptr(ptr)); + QI_IGNORE_UNUSED(fut); + } +}; } // namespace py } // namespace qi diff --git a/src/pyapplication.cpp b/src/pyapplication.cpp index 48d8052..315d842 100644 --- a/src/pyapplication.cpp +++ b/src/pyapplication.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -80,17 +81,17 @@ void exportApplication(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; class_>( m, "Application") .def(init(withArgcArgv<>([](int& argc, char**& argv) { - gil_scoped_release unlock; + GILRelease unlock; return new Application(argc, argv); })), "args"_a) - .def_static("run", &Application::run, call_guard()) - .def_static("stop", &Application::stop, call_guard()); + .def_static("run", &Application::run, call_guard()) + .def_static("stop", &Application::stop, call_guard()); class_>( @@ -98,7 +99,7 @@ void exportApplication(::py::module& m) .def(init(withArgcArgv( [](int& argc, char**& argv, bool autoExit, const std::string& url) { - gil_scoped_release unlock; + GILRelease unlock; ApplicationSession::Config config; if (!autoExit) config.setOption(qi::ApplicationSession::Option_NoAutoExit); @@ -108,22 +109,22 @@ void exportApplication(::py::module& m) })), "args"_a, "autoExit"_a, "url"_a) - .def("run", &ApplicationSession::run, call_guard(), + .def("run", &ApplicationSession::run, call_guard(), doc("Block until the end of the program (call " ":py:func:`qi.ApplicationSession.stop` to end the program).")) .def_static("stop", &ApplicationSession::stop, - call_guard(), + call_guard(), doc( "Ask the application to stop, the run function will return.")) .def("start", &ApplicationSession::startSession, - call_guard(), + call_guard(), doc("Start the connection of the session, once this function is " "called everything is fully initialized and working.")) .def_static("atRun", &ApplicationSession::atRun, - call_guard(), "func"_a, + call_guard(), "func"_a, doc( "Add a callback that will be executed when run() is called.")) @@ -131,7 +132,7 @@ void exportApplication(::py::module& m) [](const ApplicationSession& app) { return app.url().str(); }, - call_guard(), + call_guard(), doc("The url given to the Application. It's the url " "used to connect the session.")) diff --git a/src/pyasync.cpp b/src/pyasync.cpp index ba87b3a..01b6310 100644 --- a/src/pyasync.cpp +++ b/src/pyasync.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -29,7 +30,7 @@ ::py::object async(::py::function pyCallback, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; qi::uint64_t usDelay = 0; if (const auto optUsDelay = extractKeywordArg(kwargs, delayArgName)) @@ -42,7 +43,7 @@ ::py::object async(::py::function pyCallback, GILGuardedObject guardKwargs(std::move(kwargs)); auto invokeCallback = [=]() mutable { - ::py::gil_scoped_acquire lock; + GILAcquire lock; auto res = ::py::object((*guardCb)(**guardArgs, ***guardKwargs)).cast(); // Release references immediately while we hold the GIL, instead of waiting // for the lambda destructor to relock the GIL. @@ -79,7 +80,7 @@ void exportAsync(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; m.def("runAsync", &async, "callback"_a, @@ -106,7 +107,7 @@ void exportAsync(::py::module& m) [](PeriodicTask& task, qi::int64_t usPeriod) { task.setPeriod(qi::MicroSeconds(usPeriod)); }, - call_guard(), "usPeriod"_a, + call_guard(), "usPeriod"_a, doc("Set the call interval in microseconds.\n" "This call will wait until next callback invocation to apply the " "change.\n" @@ -118,7 +119,7 @@ void exportAsync(::py::module& m) " task.setUsPeriod(100)\n" " task.start()\n" ":param usPeriod: the period in microseconds")) - .def("start", &PeriodicTask::start, call_guard(), + .def("start", &PeriodicTask::start, call_guard(), "immediate"_a, doc("Start the periodic task at specified period. No effect if " "already running.\n" @@ -127,7 +128,7 @@ void exportAsync(::py::module& m) ".. warning::\n" " concurrent calls to start() and stop() will result in " "undefined behavior.")) - .def("stop", &PeriodicTask::stop, call_guard(), + .def("stop", &PeriodicTask::stop, call_guard(), doc("Stop the periodic task. When this function returns, the callback " "will not be called " "anymore. Can be called from within the callback function\n" @@ -135,12 +136,12 @@ void exportAsync(::py::module& m) " concurrent calls to start() and stop() will result in " "undefined behavior.")) .def("asyncStop", &PeriodicTask::asyncStop, - call_guard(), + call_guard(), doc("Request for periodic task to stop asynchronously.\n" "Can be called from within the callback function.")) .def( "compensateCallbackTime", &PeriodicTask::compensateCallbackTime, - call_guard(), "compensate"_a, + call_guard(), "compensate"_a, doc( ":param compensate: boolean. True to activate the compensation.\n" "When compensation is activated, call interval will take into account " @@ -149,7 +150,7 @@ void exportAsync(::py::module& m) " when the callback is longer than the specified period, " "compensation will result in the callback being called successively " "without pause.")) - .def("setName", &PeriodicTask::setName, call_guard(), + .def("setName", &PeriodicTask::setName, call_guard(), "name"_a, doc("Set name for debugging and tracking purpose")) .def("isRunning", &PeriodicTask::isRunning, doc(":returns: true if task is running")) diff --git a/src/pyclock.cpp b/src/pyclock.cpp index 027a5f7..bb5a514 100644 --- a/src/pyclock.cpp +++ b/src/pyclock.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -31,7 +32,7 @@ void exportClock(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; m.def("clockNow", &now, doc(":returns: current timestamp on qi::Clock, as a number of nanoseconds")); diff --git a/src/pyexport.cpp b/src/pyexport.cpp index 1aa0aa5..3926141 100644 --- a/src/pyexport.cpp +++ b/src/pyexport.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -31,7 +32,7 @@ void exportAll(pybind11::module& module) { registerTypes(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; exportFuture(module); exportSignal(module); diff --git a/src/pyfuture.cpp b/src/pyfuture.cpp index 4c4e9af..9c473c9 100644 --- a/src/pyfuture.cpp +++ b/src/pyfuture.cpp @@ -52,10 +52,10 @@ void castIfNotVoid(const ::py::object&) {} template std::function(Args...)> toContinuation(const ::py::function& cb) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; GILGuardedObject guardedCb(cb); auto callGuardedCb = [=](Args... args) mutable { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto handleExcept = ka::handle_exception_rethrow( exceptionLogVerbose( logCategory, @@ -78,21 +78,21 @@ std::function(Args...)> toContinuation(const ::py::function& cb) void addCallback(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; fut.connect(std::move(cont)); } Future then(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; return fut.then(std::move(cont)).unwrap(); } Future andThen(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; return fut.andThen(std::move(cont)).unwrap(); } @@ -134,7 +134,7 @@ void exportFuture(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; enum_(m, "FutureState") .value("None", FutureState_None) @@ -162,30 +162,30 @@ void exportFuture(::py::module& m) } return prom; }), - call_guard(), + call_guard(), "on_cancel"_a = none(), doc(":param on_cancel: a function that will be called when a cancel is requested on the future.\n")) .def("setCanceled", &Promise::setCanceled, - call_guard(), + call_guard(), doc("Set the state of the promise to Canceled.")) .def("setError", &Promise::setError, - call_guard(), + call_guard(), "error"_a, doc("Set the error of the promise.")) .def("setValue", &Promise::setValue, - call_guard(), + call_guard(), "value"_a, doc("Set the value of the promise.")) .def("future", &Promise::future, - call_guard(), + call_guard(), doc("Get a future tied to the promise. You can get multiple futures from the same promise.")) .def("isCancelRequested", &Promise::isCancelRequested, - call_guard(), + call_guard(), doc(":returns: True if the future associated with the promise asked for cancellation.")); class_(m, "Future") @@ -194,7 +194,7 @@ void exportFuture(::py::module& m) .def("value", static_cast(&Future::value), - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Block until the future is ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" @@ -202,7 +202,7 @@ void exportFuture(::py::module& m) ":raises: a RuntimeError if the timeout is reached or the future has error.")) .def("error", &Future::error, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Block until the future is ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" @@ -212,44 +212,44 @@ void exportFuture(::py::module& m) .def("wait", static_cast(&Future::wait), - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Wait for the future to be ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" ":returns: a :data:`qi.FutureState`.")) .def("hasError", &Future::hasError, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc(":param timeout: a time in milliseconds. Optional.\n" ":returns: true iff the future has an error.\n" ":raise: a RuntimeError if the timeout is reached.")) .def("hasValue", &Future::hasValue, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc(":param timeout: a time in milliseconds. Optional.\n" ":returns: true iff the future has a value.\n" ":raise: a RuntimeError if the timeout is reached.")) .def("cancel", &Future::cancel, - call_guard(), + call_guard(), doc("Ask for cancellation.")) .def("isFinished", &Future::isFinished, - call_guard(), + call_guard(), doc("Return true if the future is not running anymore (i.e. if hasError or hasValue or isCanceled).")) .def("isRunning", &Future::isRunning, - call_guard(), + call_guard(), doc("Return true if the future is still running.")) .def("isCanceled", &Future::isCanceled, - call_guard(), + call_guard(), doc("Return true if the future is canceled.")) .def("isCancelable", [](const Future&) { return true; }, - call_guard(), + call_guard(), doc(":returns: always true, all future are cancellable now\n" ".. deprecated:: 1.5.0\n")) @@ -279,12 +279,12 @@ void exportFuture(::py::module& m) ":returns: a future that will contain the return value of the callback.")) .def("unwrap", &qi::py::unwrap, - call_guard(), + call_guard(), doc("If this is a Future of a Future of X, return a Future of X.\n\n" "The state of both futures is forwarded and cancel requests are forwarded to the appropriate future.")); m.def("futureBarrier", &qi::py::futureBarrier, - call_guard(), + call_guard(), doc("Return a future that will be set with all the futures given as argument when they are\n" " all finished. This is useful to wait for a bunch of Futures at once.\n\n" ":param futureList: A list of Futures to wait for\n" diff --git a/src/pylog.cpp b/src/pylog.cpp index 4da952b..5e935f6 100644 --- a/src/pylog.cpp +++ b/src/pylog.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,7 @@ void exportLog(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; enum_(m, "LogLevel") .value("Silent", LogLevel_Silent) @@ -39,12 +40,12 @@ void exportLog(::py::module& m) log::log(level, name.c_str(), message.c_str(), file.c_str(), func.c_str(), line); }, - call_guard(), "level"_a, "name"_a, "message"_a, + call_guard(), "level"_a, "name"_a, "message"_a, "file"_a, "func"_a, "line"_a); m.def("setFilters", [](const std::string& filters) { log::addFilters(filters); }, - call_guard(), "filters"_a, + call_guard(), "filters"_a, doc("Set log filtering options.\n" "Each rule can be:\n\n" " +CAT: enable category CAT\n\n" @@ -60,7 +61,7 @@ void exportLog(::py::module& m) ":param filters: List of rules separated by colon.")); m.def("setContext", [](int context) { qi::log::setContext(context); }, - call_guard(), "context"_a, + call_guard(), "context"_a, doc(" 1 : Verbosity \n" " 2 : ShortVerbosity \n" " 4 : Date \n" @@ -77,7 +78,7 @@ void exportLog(::py::module& m) ":param context: A bitfield (sum of described values).")); m.def("setLevel", [](LogLevel level) { log::setLogLevel(level); }, - call_guard(), "level"_a, + call_guard(), "level"_a, doc( "Sets the threshold for the logger to level. " "Logging messages which are less severe than level will be ignored. " diff --git a/src/pymodule.cpp b/src/pymodule.cpp index 4cf04cd..5465cc0 100644 --- a/src/pymodule.cpp +++ b/src/pymodule.cpp @@ -24,7 +24,7 @@ namespace ::py::object call(::py::object obj, ::py::str name, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return obj.attr(name)(*args, **kwargs); } @@ -42,7 +42,7 @@ ::py::object getModule(const std::string& name) ::py::list listModules() { - const auto modules = invokeGuarded<::py::gil_scoped_release>(&qi::listModules); + const auto modules = invokeGuarded(&qi::listModules); return castToPyObject(AnyReference::from(modules)); } @@ -52,7 +52,7 @@ void exportObjectFactory(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; m.def("module", &getModule, doc(":returns: an object that represents the requested module.\n")); diff --git a/src/pyobject.cpp b/src/pyobject.cpp index 42076ef..11598e6 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -41,7 +41,7 @@ constexpr static const auto overloadArgName = "_overload"; ::py::object call(const Object& obj, std::string funcName, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (auto optOverload = extractKeywordArg(kwargs, overloadArgName)) funcName = *optOverload; @@ -54,7 +54,7 @@ ::py::object call(const Object& obj, std::string funcName, Promise prom; { - ::py::gil_scoped_release _unlock; + GILRelease _unlock; auto metaCallFut = obj.metaCall(funcName, argsValue.asTupleValuePtr(), async ? MetaCallType_Queued : MetaCallType_Direct); @@ -77,7 +77,7 @@ std::string docString(const MetaMethod& method) void populateMethods(::py::object pyobj, const Object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& metaObj = obj.metaObject(); const auto& methodMap = metaObj.methodMap(); @@ -158,7 +158,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, QI_ASSERT_TRUE(it != cargsEnd); ++it; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::tuple args(std::distance(it, cargsEnd)); ::py::size_t i = 0; @@ -185,7 +185,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, // the number of positional parameters the function accepts. std::string methodDefaultParametersSignature(const ::py::function& method) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Returns a Signature object (see // https://docs.python.org/3/library/inspect.html#inspect.Signature). @@ -226,7 +226,7 @@ boost::optional registerMethod(DynamicObjectBuilder& gob, const ::py::function& method, std::string parametersSignature) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (boost::starts_with(name, "__")) { @@ -269,7 +269,7 @@ namespace detail boost::optional readObjectUid(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const ::py::bytes qiObjectUid = ::py::getattr(obj, qiObjectUidAttributeName, ::py::none()); if (!qiObjectUid.is_none()) return deserializeObjectUid(static_cast(qiObjectUid)); @@ -278,7 +278,7 @@ boost::optional readObjectUid(const ::py::object& obj) void writeObjectUid(const pybind11::object& obj, const ObjectUid& uid) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const ::py::bytes uidData = serializeObjectUid(uid); ::py::setattr(obj, qiObjectUidAttributeName, uidData); } @@ -309,7 +309,7 @@ ::py::object toPyObject(Object obj) Object toObject(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Check the few known Object types that a Python object can be. if (::py::isinstance(obj)) @@ -394,7 +394,7 @@ Object toObject(const ::py::object& obj) // When the GenericObject is destroyed, the reference is released. GILGuardedObject guardedObj(obj); Object anyobj = gob.object([guardedObj](GenericObject*) mutable { - ::py::gil_scoped_acquire lock; + GILAcquire lock; *guardedObj = {}; }); @@ -417,17 +417,17 @@ void exportObject(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Object", dynamic_attr()) - .def(self == self, call_guard()) - .def(self != self, call_guard()) - .def(self < self, call_guard()) - .def(self <= self, call_guard()) - .def(self > self, call_guard()) - .def(self >= self, call_guard()) - .def("__bool__", &Object::isValid, call_guard()) - .def("isValid", &Object::isValid, call_guard()) + .def(self == self, call_guard()) + .def(self != self, call_guard()) + .def(self < self, call_guard()) + .def(self <= self, call_guard()) + .def(self > self, call_guard()) + .def(self >= self, call_guard()) + .def("__bool__", &Object::isValid, call_guard()) + .def("isValid", &Object::isValid, call_guard()) .def("call", [](Object& obj, const std::string& funcName, ::py::args args, ::py::kwargs kwargs) { @@ -443,7 +443,7 @@ void exportObject(::py::module& m) "funcName"_a) .def("metaObject", [](const Object& obj) { return AnyReference::from(obj.metaObject()); }, - call_guard()); + call_guard()); // TODO: .def("post") // TODO: .def("setProperty") // TODO: .def("property") diff --git a/src/pypath.cpp b/src/pypath.cpp index 85965a1..8d02ec6 100644 --- a/src/pypath.cpp +++ b/src/pypath.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,14 +24,14 @@ void exportPath(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; - m.def("sdkPrefix", &path::sdkPrefix, call_guard(), + m.def("sdkPrefix", &path::sdkPrefix, call_guard(), doc(":returns: The SDK prefix path. It is always a complete, native " "path.\n")); m.def( - "findBin", &path::findBin, call_guard(), "name"_a, + "findBin", &path::findBin, call_guard(), "name"_a, "searchInPath"_a = false, doc("Look for a binary in the system.\n" ":param name: string. The full name of the binary, or just the name.\n" @@ -40,14 +41,14 @@ void exportPath(::py::module& m) "otherwise.")); m.def( - "findLib", &path::findLib, call_guard(), "name"_a, + "findLib", &path::findLib, call_guard(), "name"_a, doc("Look for a library in the system.\n" ":param name: string. The full name of the library, or just the name.\n" ":returns: the complete, native path to the file found. An empty string " "otherwise.")); m.def( - "findConf", &path::findConf, call_guard(), + "findConf", &path::findConf, call_guard(), "application"_a, "file"_a, "excludeUserWritablePath"_a = false, doc("Look for a configuration file in the system.\n" ":param application: string. The name of the application.\n" @@ -59,7 +60,7 @@ void exportPath(::py::module& m) "otherwise.")); m.def( - "findData", &path::findData, call_guard(), + "findData", &path::findData, call_guard(), "application"_a, "file"_a, "excludeUserWritablePath"_a = false, doc("Look for a file in all dataPaths(application) directories. Return the " "first match.\n" @@ -76,7 +77,7 @@ void exportPath(::py::module& m) [](const std::string& applicationName, const std::string& pattern) { return path::listData(applicationName, pattern); }, - call_guard(), "applicationName"_a, "pattern"_a, + call_guard(), "applicationName"_a, "pattern"_a, doc("List data files matching the given pattern in all " "dataPaths(application) directories.\n" " For each match, return the occurrence from the first dataPaths prefix." @@ -91,12 +92,12 @@ void exportPath(::py::module& m) m.def("listData", [](const std::string& applicationName) { return path::listData(applicationName); }, - call_guard(), "applicationName"_a); + call_guard(), "applicationName"_a); m.def("confPaths", [](const std::string& applicationName) { return path::confPaths(applicationName); }, - call_guard(), + call_guard(), "applicationName"_a, doc("Get the list of directories used when searching for " "configuration files for the given application.\n" @@ -108,12 +109,12 @@ void exportPath(::py::module& m) " nor that they are writable.")); m.def("confPaths", [] { return path::confPaths(); }, - call_guard()); + call_guard()); m.def("dataPaths", [](const std::string& applicationName) { return path::dataPaths(applicationName); }, - call_guard(), + call_guard(), "applicationName"_a, doc("Get the list of directories used when searching for " "configuration files for the given application.\n" @@ -125,10 +126,10 @@ void exportPath(::py::module& m) " nor that they are writable.")); m.def("dataPaths", [] { return path::dataPaths(); }, - call_guard()); + call_guard()); m.def("binPaths", [] { return path::binPaths(); }, - call_guard(), + call_guard(), doc( ":returns: The list of directories used when searching for binaries.\n" ".. warning::\n" @@ -137,44 +138,44 @@ void exportPath(::py::module& m) m.def( "libPaths", [] { return path::libPaths(); }, - call_guard(), + call_guard(), doc(":returns: The list of directories used when searching for libraries.\n\n" ".. warning::\n" " You should not assume those directories exist," " nor that they are writable.")); m.def("setWritablePath", &path::detail::setWritablePath, - call_guard(), "path"_a, + call_guard(), "path"_a, doc("Set the writable files path for users.\n" ":param path: string. A path on the system.\n" "Use an empty path to reset it to its default value.")); m.def("userWritableDataPath", &path::userWritableDataPath, - call_guard(), "applicationName"_a, "fileName"_a, + call_guard(), "applicationName"_a, "fileName"_a, doc("Get the writable data files path for users.\n" ":param applicationName: string. Name of the application.\n" ":param fileName: string. Name of the file.\n" ":returns: The file path.")); m.def("userWritableConfPath", &path::userWritableConfPath, - call_guard(), "applicationName"_a, "fileName"_a, + call_guard(), "applicationName"_a, "fileName"_a, doc("Get the writable configuration files path for users.\n" ":param applicationName: string. Name of the application.\n" ":param fileName: string. Name of the file.\n" ":returns: The file path.")); m.def("sdkPrefixes", [] { return path::detail::getSdkPrefixes(); }, - call_guard(), + call_guard(), doc("List of SDK prefixes.\n" ":returns: The list of sdk prefixes.")); m.def("addOptionalSdkPrefix", &path::detail::addOptionalSdkPrefix, - call_guard(), "prefix"_a, + call_guard(), "prefix"_a, doc("Add a new SDK path.\n" ":param: an sdk prefix.")); m.def("clearOptionalSdkPrefix", &path::detail::clearOptionalSdkPrefix, - call_guard(), + call_guard(), doc("Clear all optional sdk prefixes.")); } diff --git a/src/pyproperty.cpp b/src/pyproperty.cpp index 9e39a45..ad91c24 100644 --- a/src/pyproperty.cpp +++ b/src/pyproperty.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -27,7 +28,7 @@ pybind11::object propertyConnect(Property& prop, const pybind11::function& pyCallback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::signalConnect(prop, pyCallback, async); } @@ -36,7 +37,7 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::proxySignalConnect(prop.object, prop.propertyId, callback, async); } @@ -45,13 +46,13 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, detail::ProxyProperty::~ProxyProperty() { // The destructor can lock when waiting for callbacks to end. - ::py::gil_scoped_release unlock; + GILRelease unlock; object.reset(); } bool isProperty(const pybind11::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return ::py::isinstance(obj) || ::py::isinstance(obj); } @@ -61,36 +62,36 @@ void exportProperty(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; using PropertyPtr = std::unique_ptr; class_(m, "Property") .def(init([] { return new Property(TypeInterface::fromSignature("m")); }), - call_guard()) + call_guard()) .def(init([](const std::string& sig) { return PropertyPtr(new Property(TypeInterface::fromSignature(sig)), DeleteOutsideGIL()); }), - "signature"_a, call_guard()) + "signature"_a, call_guard()) .def("value", [](const Property& prop, bool async) { const auto fut = prop.value().async(); - gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - arg(asyncArgName) = false, call_guard(), + arg(asyncArgName) = false, call_guard(), doc("Return the value stored inside the property.")) .def("setValue", [](Property& prop, AnyValue value, bool async) { const auto fut = toFuture(prop.setValue(std::move(value)).async()); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - call_guard(), "value"_a, arg(asyncArgName) = false, + call_guard(), "value"_a, arg(asyncArgName) = false, doc("Set the value of the property.")) .def("addCallback", &propertyConnect, "cb"_a, arg(asyncArgName) = false, @@ -107,7 +108,7 @@ void exportProperty(::py::module& m) [](Property& prop, SignalLink id, bool async) { return detail::signalDisconnect(prop, id, async); }, - call_guard(), "id"_a, arg(asyncArgName) = false, + call_guard(), "id"_a, arg(asyncArgName) = false, doc("Disconnect the callback associated to id.\n\n" ":param id: the connection id returned by :method:connect or " ":method:addCallback\n" @@ -117,7 +118,7 @@ void exportProperty(::py::module& m) [](Property& prop, bool async) { return detail::signalDisconnectAll(prop, async); }, - call_guard(), arg(asyncArgName) = false, + call_guard(), arg(asyncArgName) = false, doc("Disconnect all subscribers associated to the property.\n\n" "This function should be used with caution, as it may also remove " "subscribers that were added by other callers.\n\n" @@ -127,18 +128,18 @@ void exportProperty(::py::module& m) .def("value", [](const detail::ProxyProperty& prop, bool async) { const auto fut = prop.object.property(prop.propertyId).async(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - call_guard(), arg(asyncArgName) = false) + call_guard(), arg(asyncArgName) = false) .def("setValue", [](detail::ProxyProperty& prop, object pyValue, bool async) { AnyValue value(unwrapAsRef(pyValue)); - gil_scoped_release unlock; + GILRelease unlock; const auto fut = toFuture(prop.object.setProperty(prop.propertyId, std::move(value)) .async()); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, "value"_a, arg(asyncArgName) = false) diff --git a/src/pysession.cpp b/src/pysession.cpp index 97648f4..e54c1de 100644 --- a/src/pysession.cpp +++ b/src/pysession.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -35,7 +36,7 @@ ::py::object makeSession() { constexpr const auto category = "qi.python.session.deleter"; constexpr const auto msg = "Waiting for the shared pointer destruction..."; - ::py::gil_scoped_release x; + GILRelease x; auto fut = std::async(std::launch::async, [=](std::unique_ptr) { while (!wptr.expired()) { @@ -52,7 +53,7 @@ ::py::object makeSession() auto ptr = SessionPtr(new qi::Session{}, Deleter{}); boost::get_deleter(ptr)->wptr = ka::weak_ptr(ptr); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return py::makeSession(ptr); } @@ -60,7 +61,7 @@ ::py::object makeSession() ::py::object makeSession(SessionPtr sess) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return toPyObject(sess); } @@ -68,7 +69,7 @@ void exportSession(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; m.def("Session", []{ return makeSession(); }); } diff --git a/src/pysignal.cpp b/src/pysignal.cpp index 5ae7897..aa3a864 100644 --- a/src/pysignal.cpp +++ b/src/pysignal.cpp @@ -5,11 +5,10 @@ #include #include +#include #include #include #include -#include -#include #include #include @@ -30,7 +29,7 @@ constexpr static const auto asyncArgName = "_async"; AnyReference dynamicCallFunction(const GILGuardedObject& func, const AnyReferenceVector& args) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::list pyArgs(args.size()); ::py::size_t i = 0; for (const auto& arg : args) @@ -45,7 +44,7 @@ ::py::object connect(F&& connect, const ::py::function& pyCallback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto strand = strandOfFunction(pyCallback); SignalSubscriber subscriber(AnyFunction::fromDynamicFunction( @@ -65,7 +64,7 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::proxySignalConnect(sig.object, sig.signalId, callback, async); } @@ -77,7 +76,7 @@ namespace detail detail::ProxySignal::~ProxySignal() { // The destructor can lock waiting for callbacks to end. - ::py::gil_scoped_release unlock; + GILRelease unlock; object.reset(); } @@ -86,10 +85,10 @@ ::py::object signalConnect(SignalBase& sig, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return connect( [&](const SignalSubscriber& sub) { - ::py::gil_scoped_release unlock; + GILRelease unlock; return sig.connectAsync(sub).andThen(FutureCallbackType_Sync, [](const SignalSubscriber& sub) { return sub.link(); @@ -105,7 +104,7 @@ ::py::object signalDisconnect(SignalBase& sig, SignalLink id, bool async) return AnyValue::from(success); }); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -116,7 +115,7 @@ ::py::object signalDisconnectAll(SignalBase& sig, bool async) return AnyValue::from(success); }); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -125,10 +124,10 @@ ::py::object proxySignalConnect(const AnyObject& obj, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return connect( [&](const SignalSubscriber& sub) { - ::py::gil_scoped_release unlock; + GILRelease unlock; return obj.connect(signalId, sub).async(); }, callback, async); @@ -139,10 +138,10 @@ ::py::object proxySignalDisconnect(const AnyObject& obj, bool async) { const auto fut = [&] { - ::py::gil_scoped_release unlock; + GILRelease unlock; return toFuture(obj.disconnect(id)); }(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -150,7 +149,7 @@ ::py::object proxySignalDisconnect(const AnyObject& obj, bool isSignal(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return ::py::isinstance(obj) || ::py::isinstance(obj); } @@ -160,7 +159,7 @@ void exportSignal(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; using SignalPtr = std::unique_ptr; class_(m, "Signal") @@ -171,12 +170,12 @@ void exportSignal(::py::module& m) onSub = qi::futurizeOutput(onConnect); return new Signal(signature, onSub); }), - call_guard(), + call_guard(), "signature"_a = "m", "onConnect"_a = std::function()) .def( "connect", &detail::signalConnect, - call_guard(), + call_guard(), "callback"_a, arg(asyncArgName) = false, doc( "Connect the signal to a callback, the callback will be called each " @@ -187,13 +186,13 @@ void exportSignal(::py::module& m) ":returns: the connection id of the registered callback.")) .def("disconnect", &detail::signalDisconnect, - call_guard(), "id"_a, arg(asyncArgName) = false, + call_guard(), "id"_a, arg(asyncArgName) = false, doc("Disconnect the callback associated to id.\n" ":param id: the connection id returned by connect.\n" ":returns: true on success.")) .def("disconnectAll", &detail::signalDisconnectAll, - call_guard(), arg(asyncArgName) = false, + call_guard(), arg(asyncArgName) = false, doc("Disconnect all subscribers associated to the property.\n\n" "This function should be used with caution, as it may also remove " "subscribers that were added by other callers.\n\n" @@ -203,7 +202,7 @@ void exportSignal(::py::module& m) [](Signal& sig, args pyArgs) { const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); - gil_scoped_release unlock; + GILRelease unlock; sig.trigger(args); }, doc("Trigger the signal")); @@ -220,7 +219,7 @@ void exportSignal(::py::module& m) [](detail::ProxySignal& sig, args pyArgs) { const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); - gil_scoped_release unlock; + GILRelease unlock; sig.object.metaPost(sig.signalId, args); }); } diff --git a/src/pystrand.cpp b/src/pystrand.cpp index fb0ad89..7da98c5 100644 --- a/src/pystrand.cpp +++ b/src/pystrand.cpp @@ -5,6 +5,7 @@ #include #include +#include #include qiLogCategory("qi.python.strand"); @@ -32,7 +33,7 @@ bool isUnboundMethod(const ::py::function& func, const ::py::object& object) QI_ASSERT_TRUE(func); QI_ASSERT_TRUE(object); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto dir = ::py::reinterpret_steal<::py::list>(PyObject_Dir(object.ptr())); for (const ::py::handle attrName : dir) { @@ -77,7 +78,7 @@ ::py::object getPartialSelf(const ::py::function& partialFunc) { QI_ASSERT_TRUE(partialFunc); - ::py::gil_scoped_acquire lock; + GILAcquire lock; constexpr const char* argsAttr = "args"; constexpr const char* funcAttr = "func"; @@ -147,7 +148,7 @@ ::py::object getSelf(const ::py::function& func) { QI_ASSERT_TRUE(func); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto self = getMethodSelf(func); if (!self.is_none()) return self; @@ -158,7 +159,7 @@ ::py::object getSelf(const ::py::function& func) StrandPtr strandOfFunction(const ::py::function& func) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return strandOf(getSelf(func)); } @@ -166,7 +167,7 @@ StrandPtr strandOf(const ::py::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (obj.is_none()) return {}; @@ -202,7 +203,7 @@ bool isMultithreaded(const ::py::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyqisig = ::py::getattr(obj, objectAttributeThreadingName, ::py::none()); if (pyqisig.is_none()) return false; @@ -213,7 +214,7 @@ void exportStrand(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Strand") .def(init([] { diff --git a/src/pytranslator.cpp b/src/pytranslator.cpp index 7843d35..0eeec57 100644 --- a/src/pytranslator.cpp +++ b/src/pytranslator.cpp @@ -5,6 +5,7 @@ #include #include +#include #include namespace py = pybind11; @@ -19,21 +20,21 @@ void exportTranslator(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Translator") .def(init([](const std::string& name) { return new Translator(name); }), - call_guard(), "name"_a) - .def("translate", &Translator::translate, call_guard(), + call_guard(), "name"_a) + .def("translate", &Translator::translate, call_guard(), "msg"_a, "domain"_a = "", "locale"_a = "", "context"_a = "", doc("Translate a message from a domain to a locale.")) .def("translate", &Translator::translateContext, - call_guard(), "msg"_a, "context"_a, + call_guard(), "msg"_a, "context"_a, doc("Translate a message with a context.")) .def("setCurrentLocale", &Translator::setCurrentLocale, - call_guard(), "locale"_a, doc("Set the locale.")) + call_guard(), "locale"_a, doc("Set the locale.")) .def("setDefaultDomain", &Translator::setDefaultDomain, "domain"_a, - call_guard(), doc("Set the domain.")) + call_guard(), doc("Set the domain.")) .def("addDomain", &Translator::addDomain, "domain"_a, doc("Add a new domain.")); } diff --git a/src/pytypes.cpp b/src/pytypes.cpp index fe41bfa..ec2d617 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +33,7 @@ struct ObjectDecRef ::py::handle obj; void operator()() const { - ::py::gil_scoped_acquire lock; + GILAcquire lock; obj.dec_ref(); } }; @@ -56,20 +57,20 @@ struct ValueToPyObject void visitUnknown(AnyReference value) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Encapsulate the value in Capsule. result = ::py::capsule(value.rawValue()); } void visitVoid() { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::none(); } void visitInt(int64_t value, bool isSigned, int byteSize) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // byteSize is 0 when the value is a boolean. if (byteSize == 0) result = ::py::bool_(static_cast(value)); @@ -80,13 +81,13 @@ struct ValueToPyObject void visitFloat(double value, int /*byteSize*/) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::float_(value); } void visitString(char* data, size_t len) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (!data) { @@ -121,7 +122,7 @@ struct ValueToPyObject void visitList(AnyIterator it, AnyIterator end) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::list l; for (; it != end; ++it) l.append(unwrapValue(*it)); @@ -135,7 +136,7 @@ struct ValueToPyObject void visitMap(AnyIterator it, AnyIterator end) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict d; for (; it != end; ++it) d[unwrapValue((*it)[0])] = (*it)[1]; @@ -150,7 +151,7 @@ struct ValueToPyObject const auto type = go.type; const auto ptr = type->ptrFromStorage(&go.value); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (auto obj = tryToCastObjectTo(type, ptr)) { result = *obj; @@ -167,7 +168,7 @@ struct ValueToPyObject void visitAnyObject(AnyObject& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = py::toPyObject(obj); } @@ -182,7 +183,7 @@ struct ValueToPyObject { const auto len = tuple.size(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (annotations.empty()) { // Unnamed tuple @@ -203,7 +204,7 @@ struct ValueToPyObject void visitDynamic(AnyReference pointee) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = unwrapValue(pointee); } @@ -212,7 +213,7 @@ struct ValueToPyObject /* TODO: zerocopy, sub-buffers... */ const auto dataWithSize = value.asRaw(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::reinterpret_steal<::py::object>( PyByteArray_FromStringAndSize(dataWithSize.first, dataWithSize.second)); } @@ -224,7 +225,7 @@ struct ValueToPyObject void visitOptional(AnyReference v) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = unwrapValue(v.content()); } @@ -324,7 +325,7 @@ AnyReference associateValueToObj(::py::object& obj, T value) ::py::object unwrapValue(AnyReference val) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::object result; ValueToPyObject tpo(result); typeDispatch(tpo, val); @@ -355,20 +356,20 @@ class ObjectInterfaceBase : public Interface if (ptr) return ptr; - ::py::gil_scoped_acquire lock; + GILAcquire lock; return new Storage; } void* clone(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return new Storage(asObject(&storage)); } void destroy(void* storage) override { destroyDisownedReferences(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; delete asObjectPtr(&storage); } @@ -379,7 +380,7 @@ class ObjectInterfaceBase : public Interface bool less(void* a, void* b) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& objA = asObject(&a); const auto& objB = asObject(&b); return objA < objB; @@ -391,14 +392,14 @@ class DynamicInterface : public ObjectInterfaceBaseasObjectPtr(&storage); return unwrapAsRef(*obj); } void set(void** storage, AnyReference src) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = unwrapValue(src); } }; @@ -410,7 +411,7 @@ class IntInterface : public ObjectInterfaceBase std::int64_t get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); return numericConvertBound(::py::cast(obj)); } @@ -418,7 +419,7 @@ class IntInterface : public ObjectInterfaceBase void set(void** storage, std::int64_t val) override { QI_ASSERT_NOT_NULL(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::int_(static_cast(val)); } @@ -434,7 +435,7 @@ class FloatInterface : public ObjectInterfaceBaseasObject(&storage); return numericConvertBound(::py::cast(obj)); } @@ -442,7 +443,7 @@ class FloatInterface : public ObjectInterfaceBaseasObject(storage) = ::py::float_(static_cast(val)); } @@ -455,7 +456,7 @@ class BoolInterface : public ObjectInterfaceBase public: int64_t get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); return static_cast(::py::cast(obj)); } @@ -463,7 +464,7 @@ class BoolInterface : public ObjectInterfaceBase void set(void** storage, int64_t val) override { QI_ASSERT_NOT_NULL(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::bool_(static_cast(val)); } @@ -477,14 +478,14 @@ class StrInterface : public ObjectInterfaceBaseasObject(&storage); return makeManagedString(std::string(obj)); } void set(void** storage, const char* ptr, size_t sz) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::str(ptr, sz); } }; @@ -496,7 +497,7 @@ class StringBufferInterface : public ObjectInterfaceBaseasObject(&storage); const auto info = obj.request(); QI_ASSERT_TRUE(info.ndim == 1); @@ -506,7 +507,7 @@ class StringBufferInterface : public ObjectInterfaceBaseasObject(storage) = ::py::bytes(ptr, sz); } }; @@ -527,7 +528,7 @@ class StructuredIterableInterface std::vector get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); std::vector res; @@ -544,7 +545,7 @@ class StructuredIterableInterface { QI_ASSERT_TRUE(index < _size); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); // AppleClang 8 wrongly requires a ForwardIterator on `std::next`, which // `pybind11::iterator` is not. We use advance instead. @@ -584,7 +585,7 @@ class ListInterface : public ObjectInterfaceBase auto* listStorage = iter.first; const auto index = iter.second; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ListType list = listType->asObject(&listStorage); const ::py::object element = list[index]; @@ -625,14 +626,14 @@ class ListInterface : public ObjectInterfaceBase size_t size(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ListType list = this->asObject(&storage); return list.size(); } void pushBack(void** storage, void* valueStorage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::object obj = this->asObject(storage); if (::py::isinstance<::py::list>(obj)) { @@ -677,7 +678,7 @@ class DictInterface: public ObjectInterfaceBase auto* dictStorage = iter.first; const auto index = iter.second; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = dictType->asObject(&dictStorage); // AppleClang 8 wrongly requires a ForwardIterator on `std::next`, which // `pybind11::iterator` is not. We use advance instead. @@ -729,14 +730,14 @@ class DictInterface: public ObjectInterfaceBase size_t size(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(&storage); return dict.size(); } AnyIterator begin(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(&storage); return AnyValue(AnyReference(instance(), new Iterator(storage, 0)), // Do not copy, but free the value, so basically the AnyValue @@ -755,7 +756,7 @@ class DictInterface: public ObjectInterfaceBase void insert(void** storage, void* keyStorage, void* valueStorage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(storage); ::py::object key = keyType()->asObject(&keyStorage); ::py::object value = elementType()->asObject(&valueStorage); @@ -764,7 +765,7 @@ class DictInterface: public ObjectInterfaceBase AnyReference element(void** storage, void* keyStorage, bool autoInsert) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(storage); ::py::object key = keyType()->asObject(&keyStorage); @@ -792,7 +793,7 @@ AnyReference unwrapAsRef(pybind11::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (obj.is_none()) // The "void" value in AnyValue has no storage, so we can just release it diff --git a/tests/common.hpp b/tests/common.hpp index 2773ca8..fb4f11c 100644 --- a/tests/common.hpp +++ b/tests/common.hpp @@ -9,6 +9,7 @@ #define QIPYTHON_TESTS_COMMON_HPP #include +#include #include #include #include @@ -44,25 +45,25 @@ struct Execute { Execute() { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; _locals.emplace(); } ~Execute() { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; _locals.reset(); } void exec(const std::string& str) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; pybind11::exec(str, pybind11::globals(), *_locals); } pybind11::object eval(const std::string& str) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; return pybind11::eval(str, pybind11::globals(), *_locals); } diff --git a/tests/test_guard.cpp b/tests/test_guard.cpp index 40b261a..bc16e48 100644 --- a/tests/test_guard.cpp +++ b/tests/test_guard.cpp @@ -102,3 +102,21 @@ TEST_F(GuardedTest, GuardsCopyAssignment) EXPECT_EQ(42, a->i); EXPECT_EQ(42, b->i); } + +TEST(GILAcquire, IsReentrant) +{ + qi::py::GILAcquire acq0; QI_IGNORE_UNUSED(acq0); + qi::py::GILRelease rel; QI_IGNORE_UNUSED(rel); + qi::py::GILAcquire acq1; QI_IGNORE_UNUSED(acq1); + qi::py::GILAcquire acq2; QI_IGNORE_UNUSED(acq2); + SUCCEED(); +} + +TEST(GILRelease, IsReentrant) +{ + qi::py::GILRelease rel0; QI_IGNORE_UNUSED(rel0); + qi::py::GILAcquire acq; QI_IGNORE_UNUSED(acq); + qi::py::GILRelease rel1; QI_IGNORE_UNUSED(rel1); + qi::py::GILRelease rel2; QI_IGNORE_UNUSED(rel2); + SUCCEED(); +} diff --git a/tests/test_module.cpp b/tests/test_module.cpp index 0a11339..95b8b13 100644 --- a/tests/test_module.cpp +++ b/tests/test_module.cpp @@ -4,6 +4,7 @@ #include #include #include +#include qiLogCategory("TestQiPython.Module"); @@ -14,7 +15,7 @@ namespace void globalExec(const std::string& str) { - py::gil_scoped_acquire _lock; + qi::py::GILAcquire _lock; py::exec(str.c_str()); } @@ -42,7 +43,7 @@ MATCHER(ModuleEq, "") { return sameModule(std::get<0>(arg), std::get<1>(arg)); } TEST(Module, listModules) { - py::gil_scoped_acquire _lock; + qi::py::GILAcquire _lock; auto qi = py::globals()["qi"]; auto modulesFromPython = qi::AnyReference::from(qi.attr("listModules")()).to>(); diff --git a/tests/test_object.cpp b/tests/test_object.cpp index 761310a..a3768f7 100644 --- a/tests/test_object.cpp +++ b/tests/test_object.cpp @@ -39,7 +39,7 @@ struct ReadObjectUidTest : qi::py::test::Execute, testing::Test {}; TEST_F(ReadObjectUidTest, ReturnsEmptyIfAbsent) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto a = eval("object()"); auto objectUid = qi::py::detail::readObjectUid(a); EXPECT_FALSE(objectUid); @@ -47,7 +47,7 @@ TEST_F(ReadObjectUidTest, ReturnsEmptyIfAbsent) TEST_F(ReadObjectUidTest, ReturnsValueIfPresent) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(declareTypeAWithUid); auto a = eval("A()"); auto objectUid = qi::py::detail::readObjectUid(a); @@ -70,7 +70,7 @@ constexpr const std::array WriteObjectUidTest::data; TEST_F(WriteObjectUidTest, CanBeReadAfterWritten) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(R"( class A(object): pass @@ -85,7 +85,7 @@ class A(object): TEST_F(WriteObjectUidTest, CanBeReadAfterOverwritten) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(declareTypeAWithUid); auto a = eval("A()"); @@ -144,7 +144,7 @@ class Cookies(object): template qi::AnyObject makeObject(Args&&... args) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto type = locals()["Cookies"]; return qi::py::toObject(type(std::forward(args)...)); } @@ -216,14 +216,14 @@ struct ObjectTest : testing::Test { object = boost::make_shared(); - py::gil_scoped_acquire lock; + GILAcquire lock; pyObject = qi::py::toPyObject(object); ASSERT_TRUE(pyObject); } void TearDown() override { - py::gil_scoped_acquire lock; + GILAcquire lock; pyObject.release().dec_ref(); } @@ -248,7 +248,7 @@ QI_REGISTER_OBJECT(ObjectTest::Muffins, count, bakedCount, baked) TEST_F(ObjectTest, IsValid) { - py::gil_scoped_acquire lock; + GILAcquire lock; { EXPECT_TRUE(py::bool_(pyObject)); EXPECT_TRUE(py::bool_(pyObject.attr("isValid")())); @@ -265,18 +265,18 @@ TEST_F(ObjectTest, IsValid) TEST_F(ObjectTest, Call) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto res = pyObject.attr("call")("count", 832).cast(); EXPECT_EQ("You have 832 muffins.", res); } TEST_F(ObjectTest, Async) { - py::gil_scoped_acquire lock; + GILAcquire lock; { const auto res = qi::py::test::toFutOf( py::cast(pyObject.attr("async")("count", 2356))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 2356 muffins.", res.value()); } @@ -284,7 +284,7 @@ TEST_F(ObjectTest, Async) { const auto res = qi::py::test::toFutOf(py::cast( pyObject.attr("call")("count", 32897, py::arg("_async") = true))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 32897 muffins.", res.value()); } @@ -294,7 +294,7 @@ using ToPyObjectTest = ObjectTest; TEST_F(ToPyObjectTest, MethodCanBeCalled) { - py::gil_scoped_acquire lock; + GILAcquire lock; { const auto res = pyObject.attr("count")(8).cast(); EXPECT_EQ("You have 8 muffins.", res); @@ -305,7 +305,7 @@ TEST_F(ToPyObjectTest, MethodCanBeCalled) const auto res = qi::py::test::toFutOf(py::cast( pyObject.attr("count")(5, py::arg("_async") = true))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 5 muffins.", res.value()); } @@ -313,21 +313,21 @@ TEST_F(ToPyObjectTest, MethodCanBeCalled) TEST_F(ToPyObjectTest, PropertyIsExposed) { - py::gil_scoped_acquire lock; + GILAcquire lock; const py::object prop = pyObject.attr("bakedCount"); EXPECT_TRUE(qi::py::isProperty(prop)); } TEST_F(ToPyObjectTest, SignalIsExposed) { - py::gil_scoped_acquire lock; + GILAcquire lock; const py::object sig = pyObject.attr("baked"); EXPECT_TRUE(qi::py::isSignal(sig)); } TEST_F(ToPyObjectTest, FutureAsObjectIsReturnedAsPyFuture) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto res = pyObject.attr("count")(8).cast(); EXPECT_EQ("You have 8 muffins.", res); } diff --git a/tests/test_property.cpp b/tests/test_property.cpp index 1b74acf..03d3ee5 100644 --- a/tests/test_property.cpp +++ b/tests/test_property.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include "common.hpp" @@ -17,13 +18,13 @@ struct PropertyTest : qi::py::test::Execute, testing::Test { PropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type = py::globals()["qi"].attr("Property"); } ~PropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type.release().dec_ref(); } @@ -32,7 +33,7 @@ struct PropertyTest : qi::py::test::Execute, testing::Test TEST_F(PropertyTest, DefaultConstructedHasDynamicSignature) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyProp = type(); const auto prop = pyProp.cast(); const auto sigChildren = prop->signature().children(); @@ -42,7 +43,7 @@ TEST_F(PropertyTest, DefaultConstructedHasDynamicSignature) TEST_F(PropertyTest, ConstructedWithSignature) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyProp = type(qi::Signature::fromType(qi::Signature::Type_String).toString()); const auto prop = pyProp.cast(); @@ -55,14 +56,14 @@ struct ConstructedPropertyTest : PropertyTest { ConstructedPropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; pyProp = type("i"); prop = pyProp.cast(); } ~ConstructedPropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; prop = nullptr; pyProp = {}; } @@ -78,7 +79,7 @@ TEST_F(PropertyValueTest, CanBeReadAfterSetFromCxx) auto setFut = prop->setValue(839).async(); ASSERT_TRUE(test::finishesWithValue(setFut)); - py::gil_scoped_acquire lock; + GILAcquire lock; const auto value = pyProp.attr("value")().cast(); EXPECT_EQ(839, value); } @@ -89,7 +90,7 @@ TEST_F(PropertyValueTest, ValueCanBeReadAfterSetFromCxxAsync) ASSERT_TRUE(test::finishesWithValue(setFut)); const auto valueFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp.attr("value")(py::arg("_async") = true).cast()); }(); @@ -103,7 +104,7 @@ using PropertySetValueTest = ConstructedPropertyTest; TEST_F(PropertySetValueTest, ValueCanBeReadFromCxx) { { - py::gil_scoped_acquire lock; + GILAcquire lock; pyProp.attr("setValue")(4893); } @@ -119,7 +120,7 @@ namespace TEST_F(PropertySetValueTest, ValueCanBeReadFromCxxAsync) { const auto setValueFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp.attr("setValue")(3423409, py::arg("_async") = true) .cast()); @@ -140,7 +141,7 @@ TEST_F(PropertyAddCallbackTest, CallbackIsCalledWhenValueIsSet) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(mockFn.AsStdFunction()).cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -159,7 +160,7 @@ TEST_F(PropertyAddCallbackTest, PythonCallbackIsCalledWhenValueIsSet) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; auto pyFn = eval("lambda f : lambda i : f(i * 2)"); const auto id = pyProp.attr("addCallback")(pyFn(mockFn.AsStdFunction())) .cast(); @@ -179,7 +180,7 @@ TEST_F(PropertyAddCallbackTest, CallbackIsCalledWhenValueIsSetAsync) StrictMock> mockFn; const auto idFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp .attr("addCallback")(mockFn.AsStdFunction(), py::arg("_async") = true) @@ -201,7 +202,7 @@ TEST_F(PropertyAddCallbackTest, CallbackThrowsWhenCalledDoesNotReportError) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(mockFn.AsStdFunction()).cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -214,7 +215,7 @@ TEST_F(PropertyAddCallbackTest, CallbackThrowsWhenCalledDoesNotReportError) ASSERT_TRUE(test::finishesWithValue(setValueFut)); ASSERT_TRUE(test::finishesWithValue(cp.fut())); - py::gil_scoped_acquire lock; + GILAcquire lock; EXPECT_EQ(nullptr, PyErr_Occurred()); } @@ -222,7 +223,7 @@ using PropertyDisconnectTest = ConstructedPropertyTest; TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnected) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(py::cpp_function([](int) {})) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -233,7 +234,7 @@ TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnected) TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnectedAsync) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(py::cpp_function([](int) {})) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -246,7 +247,7 @@ TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnectedAsync) TEST_F(PropertyDisconnectTest, DisconnectRandomSignalLinkFails) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = qi::SignalLink(42); auto success = pyProp.attr("disconnect")(id).cast(); diff --git a/tests/test_qipython.cpp b/tests/test_qipython.cpp index 157226d..c6ce81f 100644 --- a/tests/test_qipython.cpp +++ b/tests/test_qipython.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ int main(int argc, char **argv) int ret = EXIT_FAILURE; { - py::gil_scoped_release unlock; + qi::py::GILRelease unlock; ret = RUN_ALL_TESTS(); // Destroy the application outside of the GIL to avoid deadlocks, but while diff --git a/tests/test_signal.cpp b/tests/test_signal.cpp index 001eaec..1bec90c 100644 --- a/tests/test_signal.cpp +++ b/tests/test_signal.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -18,7 +19,7 @@ struct DefaultConstructedSignalTest : qi::py::test::Execute, testing::Test { DefaultConstructedSignalTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type = py::globals()["qi"].attr("Signal"); pySig = type(); sig = pySig.cast(); @@ -26,7 +27,7 @@ struct DefaultConstructedSignalTest : qi::py::test::Execute, testing::Test ~DefaultConstructedSignalTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; sig = nullptr; pySig = {}; type = {}; @@ -43,7 +44,7 @@ TEST_F(DefaultConstructedSignalTest, AcceptsCallbackWithoutArgument) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pySig.attr("connect")(mockFn.AsStdFunction()) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -61,7 +62,7 @@ TEST_F(DefaultConstructedSignalTest, AcceptsCallbackWithAnyArgument) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pySig.attr("connect")(mockFn.AsStdFunction()) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -90,7 +91,7 @@ struct ConstructedThroughServiceSignalTest : qi::py::test::Execute, testing::Tes test::finishesWithValue(clientSession->connect(servSession->url()))); { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(R"py( class Cookies(object): def __init__(self): @@ -113,7 +114,7 @@ class Cookies(object): void TearDown() override { - py::gil_scoped_acquire lock; + GILAcquire lock; pySig = {}; } @@ -140,7 +141,7 @@ TEST_F(ConstructedThroughServiceSignalTest, })))); { - py::gil_scoped_acquire lock; + GILAcquire lock; pySig(); } diff --git a/tests/test_types.cpp b/tests/test_types.cpp index 155e947..5eae486 100644 --- a/tests/test_types.cpp +++ b/tests/test_types.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "common.hpp" @@ -25,14 +26,14 @@ struct PybindObjectCast { qi::AnyValue operator()(py::object obj) { - py::gil_scoped_acquire lock; + qi::py::GILAcquire lock; return obj.cast(); } }; template -struct ToAnyValueConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueConversionTest : qi::py::GILAcquire, testing::Test { template qi::AnyValue toAnyValue(T&& t) @@ -110,7 +111,7 @@ TYPED_TEST(ToAnyValueConversionTest, PyBytes) EXPECT_TRUE(v.template to().equal(py::bytes("donuts"))); } -struct ToAnyValueObjectConversionTest : py::gil_scoped_acquire, testing::Test {}; +struct ToAnyValueObjectConversionTest : qi::py::GILAcquire, testing::Test {}; TEST_F(ToAnyValueObjectConversionTest, AnyValueFromReturnsDynamic) { @@ -123,7 +124,7 @@ TEST_F(ToAnyValueObjectConversionTest, AnyValueFromReturnsDynamic) // to `PybindObjectCast` conversion with the underlying object which should // already be tested. -struct ToAnyValueListConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueListConversionTest : qi::py::GILAcquire, testing::Test { static std::vector toVec(qi::AnyReference listRef) { @@ -157,7 +158,7 @@ TEST_F(ToAnyValueListConversionTest, PybindObjectCastReturnsList) using DictTypes = testing::Types; template -struct ToAnyValueDictConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueDictConversionTest : qi::py::GILAcquire, testing::Test { static std::map toMap(qi::AnyReference map) { @@ -198,7 +199,7 @@ TYPED_TEST(ToAnyValueDictConversionTest, PybindObjectCastReturnsMap) using TupleTypes = testing::Types; template -struct ToAnyValueTupleConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueTupleConversionTest : qi::py::GILAcquire, testing::Test { static std::tuple toTuple(qi::AnyReference tuple) { @@ -241,7 +242,7 @@ struct TypePassing : qi::py::test::Execute, session = qi::makeSession(); session->listenStandalone("tcp://127.0.0.1:0"); - py::gil_scoped_acquire lock; + qi::py::GILAcquire lock; locals()["sd"] = qi::py::makeSession(session); } From a072133a9e399cd49f7cde5f578a011b2894c915 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 14 Jan 2022 20:47:31 +0100 Subject: [PATCH 07/40] qipython: Fixes an object lifetime issue causing a memory leak. #46735 When an `AnyObject` was exposed to Python as a `py::object`, we previously created wrappers for its properties and signals, that held the `AnyObject`. This caused a dependency cycle: the object maintained the properties/signals alive and the properties/signals maintained the object alive. This led to a memory leak. This patch changes the properties/signals wrapper types, so that they hold a weak reference to the object instead. The resulting dependency graph is that the `AnyObject` Python object holds its properties/signals Python objects and not the reverse, thus breaking the cycle. However, this change in turn caused a regression in `ApplicationSession`. The `ApplicationSession::session()` member function was exposed as a Python read-only property of the Python `Application` type. The property getter C++ implementation transformed the `Session` object into a temporary `AnyObject`, which was then converted into a Python object. Every access to the property therefore returned a new temporary Python object. These objects previously leaked (because of the behavior described above) and were therefore kept accessible. Because of the leak fix, they are not held anymore and are immediately destroyed unless stored into a variable. Consequently this patch also reworks the `ApplicationSession` type exposed in Python so that it now holds the Python object that holds the `AnyObject` which itself holds the `Session`. This ensures that this `Session` Python object is persistent as long as the `Application` instance is alive. Change-Id: Id1d39be6ced9208a23ae53707d1ef4d466029c68 Reviewed-on: http://gerrit2.aldebaran.lan/1784 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- qipython/pyproperty.hpp | 4 +--- qipython/pysignal.hpp | 39 +++++++++++++++++++++++++++----- src/pyapplication.cpp | 49 ++++++++++++++++++++++++++++++++++------- src/pyproperty.cpp | 18 ++++++--------- src/pysignal.cpp | 31 +++++++++++++++++--------- 5 files changed, 103 insertions(+), 38 deletions(-) diff --git a/qipython/pyproperty.hpp b/qipython/pyproperty.hpp index 2eadde8..c0f2cda 100644 --- a/qipython/pyproperty.hpp +++ b/qipython/pyproperty.hpp @@ -24,10 +24,8 @@ namespace detail { struct ProxyProperty { - AnyObject object; + AnyWeakObject object; unsigned int propertyId; - - ~ProxyProperty(); }; } diff --git a/qipython/pysignal.hpp b/qipython/pysignal.hpp index a7a9d6e..61658cd 100644 --- a/qipython/pysignal.hpp +++ b/qipython/pysignal.hpp @@ -25,12 +25,41 @@ using SignalPtr = std::shared_ptr; namespace detail { +/// The destructor of `AnyObject` can block waiting for callbacks to end. +/// This type ensures that it's always called while the GIL is unlocked. +struct AnyObjectGILSafe +{ + AnyObjectGILSafe() = default; + + /// @throws `std::runtime_error` if the weak object is expired. + /// @post With s = AnyObjectGILSafe(weak), s->isValid() + explicit AnyObjectGILSafe(AnyWeakObject weak); + + // Copy/Move operations are safe to default because they will never cause the + // destruction of the `AnyObject` contained in either the source or the + // destination (this) object. + // The declaration are still kept to enforce the 3/5/0 rule. + AnyObjectGILSafe(const AnyObjectGILSafe& o) = default; + AnyObjectGILSafe& operator=(const AnyObjectGILSafe& o) = default; + AnyObjectGILSafe(AnyObjectGILSafe&& o) = default; + AnyObjectGILSafe& operator=(AnyObjectGILSafe&& o) = default; + + ~AnyObjectGILSafe(); + + inline AnyObject& operator*() { return _obj; } + inline const AnyObject& operator*() const { return _obj; } + + inline AnyObject* operator->() { return &_obj; } + inline const AnyObject* operator->() const { return &_obj; } + +private: + AnyObject _obj; +}; + struct ProxySignal { - AnyObject object; + AnyWeakObject object; unsigned int signalId; - - ~ProxySignal(); }; pybind11::object signalConnect(SignalBase& sig, @@ -41,12 +70,12 @@ pybind11::object signalDisconnect(SignalBase& sig, SignalLink id, bool async); pybind11::object signalDisconnectAll(SignalBase& sig, bool async); -pybind11::object proxySignalConnect(const AnyObject& obj, +pybind11::object proxySignalConnect(const AnyObjectGILSafe& obj, unsigned int signalId, const pybind11::function& callback, bool async); -pybind11::object proxySignalDisconnect(const AnyObject& obj, +pybind11::object proxySignalDisconnect(const AnyObjectGILSafe& obj, SignalLink id, bool async); } // namespace detail diff --git a/src/pyapplication.cpp b/src/pyapplication.cpp index 315d842..beb3e68 100644 --- a/src/pyapplication.cpp +++ b/src/pyapplication.cpp @@ -74,6 +74,45 @@ WithArgcArgv, ExtraArgs...> withArgcArgv(F&& f) return { std::forward(f) }; } +// Wrapper for the `qi::ApplicationSession` class. +// +// Stores the `::py::object` that represents the associated `Session` object. +// +// This `::py::object` holds an `AnyObject` that wraps the `Session` object. +// Maintaining the `AnyObject` is necessary so that any properties or signals +// wrappers that the `::py::object` also holds for this qi Object don't hold +// an expired pointer to the `AnyObject`. +class ApplicationSession : private qi::ApplicationSession +{ +public: + using Base = qi::ApplicationSession; + using Config = Base::Config; + + template + explicit ApplicationSession(Args&&... args) + : Base(std::forward(args)...) + , _session(py::makeSession(Base::session())) + { + } + + ~ApplicationSession() + { + GILAcquire lock; + _session.release().dec_ref(); + } + + using Base::stop; + using Base::atRun; + + void run() { return Base::run(); } + void startSession() { return Base::startSession(); } + std::string url() const { return Base::url().str(); } + ::py::object session() const { return _session; } + +private: + ::py::object _session; +}; + } // namespace void exportApplication(::py::module& m) @@ -128,18 +167,12 @@ void exportApplication(::py::module& m) doc( "Add a callback that will be executed when run() is called.")) - .def_property_readonly("url", - [](const ApplicationSession& app) { - return app.url().str(); - }, + .def_property_readonly("url", &ApplicationSession::url, call_guard(), doc("The url given to the Application. It's the url " "used to connect the session.")) - .def_property_readonly("session", - [](const ApplicationSession& app) { - return makeSession(app.session()); - }, + .def_property_readonly("session", &ApplicationSession::session, doc("The session associated to the application.")); } diff --git a/src/pyproperty.cpp b/src/pyproperty.cpp index ad91c24..9b66563 100644 --- a/src/pyproperty.cpp +++ b/src/pyproperty.cpp @@ -38,18 +38,12 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, bool async) { GILAcquire lock; - return detail::proxySignalConnect(prop.object, prop.propertyId, callback, async); + detail::AnyObjectGILSafe object(prop.object); + return detail::proxySignalConnect(object, prop.propertyId, callback, async); } } // namespace -detail::ProxyProperty::~ProxyProperty() -{ - // The destructor can lock when waiting for callbacks to end. - GILRelease unlock; - object.reset(); -} - bool isProperty(const pybind11::object& obj) { GILAcquire lock; @@ -127,7 +121,8 @@ void exportProperty(::py::module& m) class_(m, "_ProxyProperty") .def("value", [](const detail::ProxyProperty& prop, bool async) { - const auto fut = prop.object.property(prop.propertyId).async(); + detail::AnyObjectGILSafe object(prop.object); + const auto fut = object->property(prop.propertyId).async(); GILAcquire lock; return resultObject(fut, async); }, @@ -136,8 +131,9 @@ void exportProperty(::py::module& m) [](detail::ProxyProperty& prop, object pyValue, bool async) { AnyValue value(unwrapAsRef(pyValue)); GILRelease unlock; + detail::AnyObjectGILSafe object(prop.object); const auto fut = - toFuture(prop.object.setProperty(prop.propertyId, std::move(value)) + toFuture(object->setProperty(prop.propertyId, std::move(value)) .async()); GILAcquire lock; return resultObject(fut, async); @@ -149,7 +145,7 @@ void exportProperty(::py::module& m) arg(asyncArgName) = false) .def("disconnect", [](detail::ProxyProperty& prop, SignalLink id, bool async) { - return detail::proxySignalDisconnect(prop.object, id, async); + return detail::proxySignalDisconnect(detail::AnyObjectGILSafe(prop.object), id, async); }, "id"_a, arg(asyncArgName) = false); } diff --git a/src/pysignal.cpp b/src/pysignal.cpp index aa3a864..6179bcd 100644 --- a/src/pysignal.cpp +++ b/src/pysignal.cpp @@ -65,7 +65,7 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, bool async) { GILAcquire lock; - return detail::proxySignalConnect(sig.object, sig.signalId, callback, async); + return detail::proxySignalConnect(detail::AnyObjectGILSafe(sig.object), sig.signalId, callback, async); } } // namespace @@ -73,13 +73,20 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, namespace detail { -detail::ProxySignal::~ProxySignal() +static constexpr const auto objectExpiredMsg = "object has expired"; + +AnyObjectGILSafe::AnyObjectGILSafe(AnyWeakObject weakObj) { - // The destructor can lock waiting for callbacks to end. - GILRelease unlock; - object.reset(); + _obj = weakObj.lock(); + if (!_obj.isValid()) + throw std::runtime_error(objectExpiredMsg); } +AnyObjectGILSafe::~AnyObjectGILSafe() +{ + GILRelease unlock; + _obj = {}; +} ::py::object signalConnect(SignalBase& sig, const ::py::function& callback, @@ -119,7 +126,7 @@ ::py::object signalDisconnectAll(SignalBase& sig, bool async) return resultObject(fut, async); } -::py::object proxySignalConnect(const AnyObject& obj, +::py::object proxySignalConnect(const AnyObjectGILSafe& obj, unsigned int signalId, const ::py::function& callback, bool async) @@ -128,18 +135,18 @@ ::py::object proxySignalConnect(const AnyObject& obj, return connect( [&](const SignalSubscriber& sub) { GILRelease unlock; - return obj.connect(signalId, sub).async(); + return obj->connect(signalId, sub).async(); }, callback, async); } -::py::object proxySignalDisconnect(const AnyObject& obj, +::py::object proxySignalDisconnect(const AnyObjectGILSafe& obj, SignalLink id, bool async) { const auto fut = [&] { GILRelease unlock; - return toFuture(obj.disconnect(id)); + return toFuture(obj->disconnect(id)); }(); GILAcquire lock; return resultObject(fut, async); @@ -212,7 +219,8 @@ void exportSignal(::py::module& m) arg(asyncArgName) = false) .def("disconnect", [](detail::ProxySignal& sig, SignalLink id, bool async) { - return detail::proxySignalDisconnect(sig.object, id, async); + auto object = detail::AnyObjectGILSafe(sig.object); + return detail::proxySignalDisconnect(object, id, async); }, "id"_a, arg(asyncArgName) = false) .def("__call__", @@ -220,7 +228,8 @@ void exportSignal(::py::module& m) const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); GILRelease unlock; - sig.object.metaPost(sig.signalId, args); + auto object = detail::AnyObjectGILSafe(sig.object); + object->metaPost(sig.signalId, args); }); } From d009f1df6227686d1a3193dead0d92409c4a73a3 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Thu, 18 Nov 2021 14:38:56 +0100 Subject: [PATCH 08/40] build: Upgrades version of Boost from 1.64 to 1.77. Change-Id: Ibd9f299e0a425962ab34804fc4e570dd2c21e17a Reviewed-on: http://gerrit2.aldebaran.lan/1680 Reviewed-by: jmonnon Reviewed-by: philippe.martin Tested-by: vincent.palancher --- cmake/set_dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake index dab180e..d5996d2 100644 --- a/cmake/set_dependencies.cmake +++ b/cmake/set_dependencies.cmake @@ -45,7 +45,7 @@ include_guard(GLOBAL) # Version of Boost to use. It will be used as the argument to the `find_package(Boost)` call. -overridable_variable(BOOST_VERSION 1.64) +overridable_variable(BOOST_VERSION 1.77) # Version of pybind11 to use. overridable_variable(PYBIND11_VERSION 2.5.0) From cf0ce2fa8815eccf34e9533b41ec4db1d038be8a Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 11 Jan 2022 17:17:59 +0100 Subject: [PATCH 09/40] build: Upgrades version of googletest to live at head. The tagged versions of Googletest are considered obsolete. The recommended version is the one targeted by the `main` branch. This patch adds an include to `optional_io.hpp`, because a change in googletest caused one of our tests to try to output a `boost::optional`. Integration breaking change: the GOOGLETEST_VERSION CMake variable is no longer honored. Change-Id: Ie816821af53a79842ddad4f2e5c56e26a2e20782 Reviewed-on: http://gerrit2.aldebaran.lan/1761 Reviewed-by: philippe.martin Reviewed-by: jmonnon Tested-by: vincent.palancher --- cmake/set_dependencies.cmake | 5 +---- tests/test_object.cpp | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake index d5996d2..b61a427 100644 --- a/cmake/set_dependencies.cmake +++ b/cmake/set_dependencies.cmake @@ -58,16 +58,13 @@ overridable_variable(PYBIND11_GIT_REPOSITORY https://github.com/pybind/pybind11) # `ExternalProject` module documentation of the `GIT_TAG` argument. overridable_variable(PYBIND11_GIT_TAG v${PYBIND11_VERSION}) -# Version of googletest to use. -overridable_variable(GOOGLETEST_VERSION 1.10.0) - # URL of the git repository from which to download googletest. For more details, see CMake # `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. overridable_variable(GOOGLETEST_GIT_REPOSITORY https://github.com/google/googletest.git) # Git branch name, tag or commit hash to checkout for googletest. For more details, see CMake # `ExternalProject` module documentation of the `GIT_TAG` argument. -overridable_variable(GOOGLETEST_GIT_TAG release-${GOOGLETEST_VERSION}) +overridable_variable(GOOGLETEST_GIT_TAG main) set(PYTHON_VERSION_STRING "" CACHE STRING "Version of Python to look for. This \ variable can be specified by tools run directly from Python to enforce the \ diff --git a/tests/test_object.cpp b/tests/test_object.cpp index a3768f7..b84dbcf 100644 --- a/tests/test_object.cpp +++ b/tests/test_object.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "common.hpp" namespace py = pybind11; From ea3f4225b7daa589893068927fd91866c8ee9072 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 11 Jan 2022 17:01:04 +0100 Subject: [PATCH 10/40] build: Upgrades version of pybind11 from v2.5.0 to v2.9.0. The public constructor of `pybind11::module` is now deprecated. This patch replaces its only use by a call to `PYBIND11_EMBEDDED_MODULE` followed by a `pybind11::import(...)`. The constructor of a `pybind11::object` subclass from a `pybind11::object` from the wrong type now throws an exception. See https://github.com/pybind/pybind11/pull/2349. We had one case of this happening where we tried constructing a `bytes` from a `None`. We now first construct an `object`, then test if it's `None` before trying to convert it to `bytes`. Change-Id: I4aa2a1b4a49d3024f203bdd5c6f9c4ee86bfc9c4 Reviewed-on: http://gerrit2.aldebaran.lan/1760 Reviewed-by: philippe.martin Reviewed-by: jmonnon Tested-by: vincent.palancher --- cmake/set_dependencies.cmake | 2 +- src/pyobject.cpp | 9 +++++---- tests/test_qipython.cpp | 9 ++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake index b61a427..ce237f0 100644 --- a/cmake/set_dependencies.cmake +++ b/cmake/set_dependencies.cmake @@ -48,7 +48,7 @@ include_guard(GLOBAL) overridable_variable(BOOST_VERSION 1.77) # Version of pybind11 to use. -overridable_variable(PYBIND11_VERSION 2.5.0) +overridable_variable(PYBIND11_VERSION 2.9.0) # URL of the git repository from which to download pybind11. For more details, see CMake # `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. diff --git a/src/pyobject.cpp b/src/pyobject.cpp index 11598e6..d90a1e0 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -270,10 +270,11 @@ namespace detail boost::optional readObjectUid(const ::py::object& obj) { GILAcquire lock; - const ::py::bytes qiObjectUid = ::py::getattr(obj, qiObjectUidAttributeName, ::py::none()); - if (!qiObjectUid.is_none()) - return deserializeObjectUid(static_cast(qiObjectUid)); - return {}; + const auto qiObjectUidObj = ::py::getattr(obj, qiObjectUidAttributeName, ::py::none()); + if (qiObjectUidObj.is_none()) + return {}; + const auto qiObjectUid = qiObjectUidObj.cast(); + return deserializeObjectUid(qiObjectUid); } void writeObjectUid(const pybind11::object& obj, const ObjectUid& uid) diff --git a/tests/test_qipython.cpp b/tests/test_qipython.cpp index c6ce81f..1ac1495 100644 --- a/tests/test_qipython.cpp +++ b/tests/test_qipython.cpp @@ -15,6 +15,10 @@ qiLogCategory("TestQiPython"); namespace py = pybind11; +PYBIND11_EMBEDDED_MODULE(qi, m) { + qi::py::exportAll(m); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); @@ -24,9 +28,8 @@ int main(int argc, char **argv) boost::optional app; app.emplace(argc, argv); - py::module m("qi"); - qi::py::exportAll(m); - py::globals()["qi"] = m; + py::globals()["qi"] = py::module::import("qi"); + int ret = EXIT_FAILURE; { From 40929b737f46531193e788f22d3e0c723b5b88fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Barth=C3=A9l=C3=A9my?= Date: Thu, 30 Sep 2021 23:03:56 +0200 Subject: [PATCH 11/40] examples: Make authentication CLI parsing non-intrusive. This ensures `make_application()` can be imported and used by different programs, which will also parse their own CLI options. Change-Id: I0df932210a2c350ce2721153494b252d4f8df89e Reviewed-on: http://gerrit2.aldebaran.lan/1562 Reviewed-by: jmonnon Reviewed-by: vincent.palancher Tested-by: vincent.palancher --- examples/authentication_with_application.py | 37 +++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/examples/authentication_with_application.py b/examples/authentication_with_application.py index 49ac4a7..003b2b2 100644 --- a/examples/authentication_with_application.py +++ b/examples/authentication_with_application.py @@ -28,7 +28,7 @@ def newAuthenticator(self): # Reads a file containing the username on the first line and the password on -# the second line. +# the second line. This is the format used by qilaunch. def read_auth_file(path): with open(path) as f: username = f.readline().strip() @@ -36,27 +36,52 @@ def read_auth_file(path): return (username, password) -def make_application(): - app = qi.Application(sys.argv) - parser = argparse.ArgumentParser() +def make_application(argv=sys.argv): + """ + Create and return the qi.Application, with authentication set up + according to the command line options. + """ + # create the app and edit `argv` in place to remove the consumed + # arguments. + # As a side effect, if "-h" is in the list, it is replaced with "--help". + app = qi.Application(argv) + + # Setup a non-intrusive parser, behaving like `qi.Application`'s own + # parser: + # * don't complain about unknown arguments + # * consume known arguments + # * if the "--help" option is present: + # * print its own options help + # * do not print the main app usage + # * do not call `sys.exit()` + parser = argparse.ArgumentParser(add_help=False, usage=argparse.SUPPRESS) parser.add_argument( "-a", "--authfile", help="Path to the authentication config file. This file must " "contain the username on the first line and the password on the " "second line.") - args = parser.parse_args(sys.argv[1:]) + if "--help" in argv: + parser.print_help() + return app + args, unparsed_args = parser.parse_known_args(argv[1:]) logins = read_auth_file(args.authfile) if args.authfile else ("nao", "nao") factory = AuthenticatorFactory(*logins) app.session.setClientAuthenticatorFactory(factory) + # edit argv in place. + # Note: this might modify sys.argv, like qi.Application does. + argv[1:] = unparsed_args return app if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--msg", default="Hello python") app = make_application() + args = parser.parse_args() logger = qi.Logger("authentication_with_application") logger.info("connecting session") app.start() logger.info("fetching ALTextToSpeech service") tts = app.session.service("ALTextToSpeech") logger.info("Saying something") - tts.call("say", "Hello python") + tts.call("say", args.msg) From 048e9435f7853b322758f611c8106c98dd304564 Mon Sep 17 00:00:00 2001 From: "philippe.martin" Date: Tue, 3 May 2022 18:02:12 +0200 Subject: [PATCH 12/40] Upgrades libqi to version 2.1.0. Change-Id: I9faed071a01b57d178052bb3c18496398bb5c992 Reviewed-on: http://gerrit2.aldebaran.lan/2002 Reviewed-by: jmonnon Reviewed-by: vincent.palancher Tested-by: philippe.martin --- cmake/set_libqi_dependency.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake index bd32edc..31679dc 100644 --- a/cmake/set_libqi_dependency.cmake +++ b/cmake/set_libqi_dependency.cmake @@ -1,4 +1,4 @@ -overridable_variable(LIBQI_VERSION 2.0.0) +overridable_variable(LIBQI_VERSION 2.1.0) # Our github clone is sometimes late or is missing tags, so we enable # customisation of the URL at configuration time, so users can use another clone. From d92be88ae1add4012f2abac64e0d5366fd00d407 Mon Sep 17 00:00:00 2001 From: "philippe.martin" Date: Tue, 3 May 2022 18:04:09 +0200 Subject: [PATCH 13/40] Bumps version to libqi-python 3.1.0. Change-Id: Id080a7fa30c40529fca5f5bf9d78358ee2f55fd2 Reviewed-on: http://gerrit2.aldebaran.lan/2003 Reviewed-by: jmonnon Reviewed-by: vincent.palancher Tested-by: philippe.martin --- qi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qi/_version.py b/qi/_version.py index 4eb28e3..7f5601d 100644 --- a/qi/_version.py +++ b/qi/_version.py @@ -1 +1 @@ -__version__ = '3.0.0' +__version__ = '3.1.0' From 5dde1d151327ccf97bdfbf52e99c3e904469e5a3 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 3 Jun 2022 20:15:44 +0200 Subject: [PATCH 14/40] qi.pyguard: Fixes segfault at Python interpreter finalization due to GIL bad manipulation. #47167 A segfault occurs during the finalization of the Python interpreter when an object has a destructor that uses `GILRelease`. The constructor of `GILRelease` effectively releases the GIL but the destructor disables its acquisition because the interpreter is finalizing. When the call returns to the interpreter, the GIL stays released and this leads to crashes or errors when the interpreter tries to make Python calls. This patch rewrites the `GILRelease` type so that during finalization, the object does nothing (therefore the GIL is not released). --- cmake/add_tests.cmake | 34 ++++++++++++------ qipython/pyguard.hpp | 42 +++++++++++------------ tests/test_qipython_local_interpreter.cpp | 41 ++++++++++++++++++++++ 3 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 tests/test_qipython_local_interpreter.cpp diff --git a/cmake/add_tests.cmake b/cmake/add_tests.cmake index ea46f38..b69faf7 100644 --- a/cmake/add_tests.cmake +++ b/cmake/add_tests.cmake @@ -49,18 +49,32 @@ target_link_libraries(test_qipython qi.interface cxx11 gmock) -# Unfortunately, in some of our toolchains, gtest/gmock headers are found in the qi-framework -# package, which comes first in the include paths order given to the compiler. This causes the -# compiler to use those headers instead of the ones we got from a `FetchContent` of the googletest -# repository. -target_include_directories(test_qipython - BEFORE # Force this path to come first in the list of include paths. - PRIVATE $) -enable_warnings(test_qipython) -set_build_rpath_to_qipython_dependencies(test_qipython) + +add_executable(test_qipython_local_interpreter) +target_sources(test_qipython_local_interpreter + PRIVATE tests/test_qipython_local_interpreter.cpp) +target_link_libraries(test_qipython_local_interpreter + PRIVATE Python::Python + pybind11 + qi_python_objects + qi.interface + cxx11 + gmock) set(_sdk_prefix "${CMAKE_BINARY_DIR}/sdk") -gtest_discover_tests(test_qipython EXTRA_ARGS --qi-sdk-prefix ${_sdk_prefix}) +foreach(test_target IN ITEMS test_qipython + test_qipython_local_interpreter) + # Unfortunately, in some of our toolchains, gtest/gmock headers are found in the qi-framework + # package, which comes first in the include paths order given to the compiler. This causes the + # compiler to use those headers instead of the ones we got from a `FetchContent` of the googletest + # repository. + target_include_directories(${test_target} + BEFORE # Force this path to come first in the list of include paths. + PRIVATE $) + enable_warnings(${test_target}) + set_build_rpath_to_qipython_dependencies(${test_target}) + gtest_discover_tests(${test_target} EXTRA_ARGS --qi-sdk-prefix ${_sdk_prefix}) +endforeach() if(NOT Python_Interpreter_FOUND) message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") diff --git a/qipython/pyguard.hpp b/qipython/pyguard.hpp index f09fcd6..33ea618 100644 --- a/qipython/pyguard.hpp +++ b/qipython/pyguard.hpp @@ -143,59 +143,57 @@ void pybind11GuardDisarm(G& guard) /// RAII utility type that guarantees that the GIL is locked for the scope of /// the lifetime of the object. /// +/// Objects of this type (or objects composed of them) must not be kept alive +/// after the hand is given back to the interpreter. +/// /// This type is re-entrant. /// -/// postcondition: `GILAcquire acq;` establishes `currentThreadHoldsGil()` +/// postcondition: `GILAcquire acq;` establishes +/// `(interpreterIsFinalizing() && *interpreterIsFinalizing()) || currentThreadHoldsGil()` struct GILAcquire { inline GILAcquire() { + const auto optIsFinalizing = interpreterIsFinalizing(); + const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; // `gil_scoped_acquire` is re-entrant by itself, so we don't need to check // whether or not the GIL is already held by the current thread. - QI_ASSERT(currentThreadHoldsGil()); + if (!definitelyFinalizing) + _acq.emplace(); + QI_ASSERT(definitelyFinalizing || currentThreadHoldsGil()); } GILAcquire(const GILAcquire&) = delete; GILAcquire& operator=(const GILAcquire&) = delete; - inline ~GILAcquire() - { - const auto optIsFinalizing = interpreterIsFinalizing(); - const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; - if (definitelyFinalizing) - detail::pybind11GuardDisarm(_acq); - } - private: - pybind11::gil_scoped_acquire _acq; + boost::optional _acq; }; /// RAII utility type that guarantees that the GIL is unlocked for the scope of /// the lifetime of the object. /// +/// Objects of this type (or objects composed of them) must not be kept alive +/// after the hand is given back to the interpreter. +/// /// This type is re-entrant. /// -/// postcondition: `GILRelease rel;` establishes `!currentThreadHoldsGil()` +/// postcondition: `GILRelease rel;` establishes +/// `(interpreterIsFinalizing() && *interpreterIsFinalizing()) || !currentThreadHoldsGil()` struct GILRelease { inline GILRelease() { - if (currentThreadHoldsGil()) + const auto optIsFinalizing = interpreterIsFinalizing(); + const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; + if (!definitelyFinalizing && currentThreadHoldsGil()) _release.emplace(); - QI_ASSERT(!currentThreadHoldsGil()); + QI_ASSERT(definitelyFinalizing || !currentThreadHoldsGil()); } GILRelease(const GILRelease&) = delete; GILRelease& operator=(const GILRelease&) = delete; - inline ~GILRelease() - { - const auto optIsFinalizing = interpreterIsFinalizing(); - const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; - if (_release && definitelyFinalizing) - detail::pybind11GuardDisarm(*_release); - } - private: boost::optional _release; }; diff --git a/tests/test_qipython_local_interpreter.cpp b/tests/test_qipython_local_interpreter.cpp new file mode 100644 index 0000000..c1113f7 --- /dev/null +++ b/tests/test_qipython_local_interpreter.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_EMBEDDED_MODULE(test_local_interpreter, m) { + struct ObjectDtorOutsideGIL + { + ~ObjectDtorOutsideGIL() + { + qi::py::GILRelease _rel; + // nothing. + } + + }; + pybind11::class_(m, "ObjectDtorOutsideGIL") + .def(pybind11::init([]{ + return std::make_unique(); + })); +} + +TEST(InterpreterFinalize, DoesNotSegfaultGarbageObjectDtorOutsideGIL) +{ + pybind11::scoped_interpreter interp; + pybind11::globals()["qitli"] = pybind11::module::import("test_local_interpreter"); + pybind11::exec("obj = qitli.ObjectDtorOutsideGIL()"); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + qi::Application app(argc, argv); + return RUN_ALL_TESTS(); +} From f96693e47a341fa00c608f5f532f1c4a01a504c5 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 7 Jun 2022 14:34:48 +0200 Subject: [PATCH 15/40] Fixes a memory leak in native code and a regression. #46735 This patch fixes a leak caused by errors in the native code and a regression caused by a previous tentative of a fix for another leak. The leak occurs in qi objects bound functions arguments conversion. It was not clear whether `qi::StructTypeInterface` subclasses implementations of its `get` methods should return owned pointers or not (i.e. if the caller of these methods was expected to destroy these pointers). It seems after research that they must not, and our `StructuredIterableInterface` subtype implementation was wrong. These methods are reimplemented by this patch. The regression was caused by a previous fix. The fix (commit "a072133") aimed at resolving a leak that concerned qi objects containing properties or signals, which were kept alive after the Python object holding them was destroyed. This fix introduced a regression that prevented users to call functions of properties from temporary objects. Therefore, this patch reverts that commit. After further research, it was found that the leak is caused by objects being instantiated on the heap when we thought `pybind11` took ownership of these, which is not the case. Instead, the objects are now instantiated on the stack. --- qi/test/test_module.py | 23 ++++++++++++++++++- qipython/pyproperty.hpp | 4 +++- qipython/pysignal.hpp | 39 +++++--------------------------- src/pyapplication.cpp | 49 +++++++---------------------------------- src/pyobject.cpp | 4 ++-- src/pyproperty.cpp | 18 +++++++++------ src/pysignal.cpp | 31 +++++++++----------------- src/pytypes.cpp | 12 ++++++---- tests/moduletest.cpp | 8 ++++++- 9 files changed, 77 insertions(+), 111 deletions(-) diff --git a/qi/test/test_module.py b/qi/test/test_module.py index 8796116..b6564e4 100644 --- a/qi/test/test_module.py +++ b/qi/test/test_module.py @@ -42,7 +42,6 @@ def test_module_service(): cat = session.service("Cat") assert cat.meow(3) == 'meow' - def test_module_service_object_lifetime(): session = qi.Session() session.listenStandalone("tcp://localhost:0") @@ -72,3 +71,25 @@ def test_module_service_object_lifetime(): assert cat.nbPlay() == 1 del play assert cat.nbPlay() == 0 + +def test_object_bound_functions_arguments_conversion_does_not_leak(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + cat = session.service("Cat") + + play = cat.makePlay() + assert cat.nbPlay() == 1 + cat.order(play) + assert cat.nbPlay() == 1 + del play + assert cat.nbPlay() == 0 + +def test_temporary_object_bound_properties_are_usable(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + # The `volume` member is a bound property of the `Purr` object. + # It should keep the object alive so that setting the property, which + # requires accessing the object, does not fail. + session.service("Cat").makePurr().volume.setValue(42) diff --git a/qipython/pyproperty.hpp b/qipython/pyproperty.hpp index c0f2cda..2eadde8 100644 --- a/qipython/pyproperty.hpp +++ b/qipython/pyproperty.hpp @@ -24,8 +24,10 @@ namespace detail { struct ProxyProperty { - AnyWeakObject object; + AnyObject object; unsigned int propertyId; + + ~ProxyProperty(); }; } diff --git a/qipython/pysignal.hpp b/qipython/pysignal.hpp index 61658cd..a7a9d6e 100644 --- a/qipython/pysignal.hpp +++ b/qipython/pysignal.hpp @@ -25,41 +25,12 @@ using SignalPtr = std::shared_ptr; namespace detail { -/// The destructor of `AnyObject` can block waiting for callbacks to end. -/// This type ensures that it's always called while the GIL is unlocked. -struct AnyObjectGILSafe -{ - AnyObjectGILSafe() = default; - - /// @throws `std::runtime_error` if the weak object is expired. - /// @post With s = AnyObjectGILSafe(weak), s->isValid() - explicit AnyObjectGILSafe(AnyWeakObject weak); - - // Copy/Move operations are safe to default because they will never cause the - // destruction of the `AnyObject` contained in either the source or the - // destination (this) object. - // The declaration are still kept to enforce the 3/5/0 rule. - AnyObjectGILSafe(const AnyObjectGILSafe& o) = default; - AnyObjectGILSafe& operator=(const AnyObjectGILSafe& o) = default; - AnyObjectGILSafe(AnyObjectGILSafe&& o) = default; - AnyObjectGILSafe& operator=(AnyObjectGILSafe&& o) = default; - - ~AnyObjectGILSafe(); - - inline AnyObject& operator*() { return _obj; } - inline const AnyObject& operator*() const { return _obj; } - - inline AnyObject* operator->() { return &_obj; } - inline const AnyObject* operator->() const { return &_obj; } - -private: - AnyObject _obj; -}; - struct ProxySignal { - AnyWeakObject object; + AnyObject object; unsigned int signalId; + + ~ProxySignal(); }; pybind11::object signalConnect(SignalBase& sig, @@ -70,12 +41,12 @@ pybind11::object signalDisconnect(SignalBase& sig, SignalLink id, bool async); pybind11::object signalDisconnectAll(SignalBase& sig, bool async); -pybind11::object proxySignalConnect(const AnyObjectGILSafe& obj, +pybind11::object proxySignalConnect(const AnyObject& obj, unsigned int signalId, const pybind11::function& callback, bool async); -pybind11::object proxySignalDisconnect(const AnyObjectGILSafe& obj, +pybind11::object proxySignalDisconnect(const AnyObject& obj, SignalLink id, bool async); } // namespace detail diff --git a/src/pyapplication.cpp b/src/pyapplication.cpp index beb3e68..315d842 100644 --- a/src/pyapplication.cpp +++ b/src/pyapplication.cpp @@ -74,45 +74,6 @@ WithArgcArgv, ExtraArgs...> withArgcArgv(F&& f) return { std::forward(f) }; } -// Wrapper for the `qi::ApplicationSession` class. -// -// Stores the `::py::object` that represents the associated `Session` object. -// -// This `::py::object` holds an `AnyObject` that wraps the `Session` object. -// Maintaining the `AnyObject` is necessary so that any properties or signals -// wrappers that the `::py::object` also holds for this qi Object don't hold -// an expired pointer to the `AnyObject`. -class ApplicationSession : private qi::ApplicationSession -{ -public: - using Base = qi::ApplicationSession; - using Config = Base::Config; - - template - explicit ApplicationSession(Args&&... args) - : Base(std::forward(args)...) - , _session(py::makeSession(Base::session())) - { - } - - ~ApplicationSession() - { - GILAcquire lock; - _session.release().dec_ref(); - } - - using Base::stop; - using Base::atRun; - - void run() { return Base::run(); } - void startSession() { return Base::startSession(); } - std::string url() const { return Base::url().str(); } - ::py::object session() const { return _session; } - -private: - ::py::object _session; -}; - } // namespace void exportApplication(::py::module& m) @@ -167,12 +128,18 @@ void exportApplication(::py::module& m) doc( "Add a callback that will be executed when run() is called.")) - .def_property_readonly("url", &ApplicationSession::url, + .def_property_readonly("url", + [](const ApplicationSession& app) { + return app.url().str(); + }, call_guard(), doc("The url given to the Application. It's the url " "used to connect the session.")) - .def_property_readonly("session", &ApplicationSession::session, + .def_property_readonly("session", + [](const ApplicationSession& app) { + return makeSession(app.session()); + }, doc("The session associated to the application.")); } diff --git a/src/pyobject.cpp b/src/pyobject.cpp index d90a1e0..ddd1a4f 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -121,7 +121,7 @@ void populateSignals(::py::object pyobj, const Object& obj) continue; ::py::setattr(pyobj, signalName.c_str(), - castToPyObject(new detail::ProxySignal{ obj, signal.uid() })); + castToPyObject(detail::ProxySignal{ obj, signal.uid() })); } } @@ -138,7 +138,7 @@ void populateProperties(::py::object pyobj, const Object& obj) continue; ::py::setattr(pyobj, propName.c_str(), - castToPyObject(new detail::ProxyProperty{ obj, prop.uid() })); + castToPyObject(detail::ProxyProperty{ obj, prop.uid() })); } } diff --git a/src/pyproperty.cpp b/src/pyproperty.cpp index 9b66563..ad91c24 100644 --- a/src/pyproperty.cpp +++ b/src/pyproperty.cpp @@ -38,12 +38,18 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, bool async) { GILAcquire lock; - detail::AnyObjectGILSafe object(prop.object); - return detail::proxySignalConnect(object, prop.propertyId, callback, async); + return detail::proxySignalConnect(prop.object, prop.propertyId, callback, async); } } // namespace +detail::ProxyProperty::~ProxyProperty() +{ + // The destructor can lock when waiting for callbacks to end. + GILRelease unlock; + object.reset(); +} + bool isProperty(const pybind11::object& obj) { GILAcquire lock; @@ -121,8 +127,7 @@ void exportProperty(::py::module& m) class_(m, "_ProxyProperty") .def("value", [](const detail::ProxyProperty& prop, bool async) { - detail::AnyObjectGILSafe object(prop.object); - const auto fut = object->property(prop.propertyId).async(); + const auto fut = prop.object.property(prop.propertyId).async(); GILAcquire lock; return resultObject(fut, async); }, @@ -131,9 +136,8 @@ void exportProperty(::py::module& m) [](detail::ProxyProperty& prop, object pyValue, bool async) { AnyValue value(unwrapAsRef(pyValue)); GILRelease unlock; - detail::AnyObjectGILSafe object(prop.object); const auto fut = - toFuture(object->setProperty(prop.propertyId, std::move(value)) + toFuture(prop.object.setProperty(prop.propertyId, std::move(value)) .async()); GILAcquire lock; return resultObject(fut, async); @@ -145,7 +149,7 @@ void exportProperty(::py::module& m) arg(asyncArgName) = false) .def("disconnect", [](detail::ProxyProperty& prop, SignalLink id, bool async) { - return detail::proxySignalDisconnect(detail::AnyObjectGILSafe(prop.object), id, async); + return detail::proxySignalDisconnect(prop.object, id, async); }, "id"_a, arg(asyncArgName) = false); } diff --git a/src/pysignal.cpp b/src/pysignal.cpp index 6179bcd..aa3a864 100644 --- a/src/pysignal.cpp +++ b/src/pysignal.cpp @@ -65,7 +65,7 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, bool async) { GILAcquire lock; - return detail::proxySignalConnect(detail::AnyObjectGILSafe(sig.object), sig.signalId, callback, async); + return detail::proxySignalConnect(sig.object, sig.signalId, callback, async); } } // namespace @@ -73,21 +73,14 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, namespace detail { -static constexpr const auto objectExpiredMsg = "object has expired"; - -AnyObjectGILSafe::AnyObjectGILSafe(AnyWeakObject weakObj) -{ - _obj = weakObj.lock(); - if (!_obj.isValid()) - throw std::runtime_error(objectExpiredMsg); -} - -AnyObjectGILSafe::~AnyObjectGILSafe() +detail::ProxySignal::~ProxySignal() { + // The destructor can lock waiting for callbacks to end. GILRelease unlock; - _obj = {}; + object.reset(); } + ::py::object signalConnect(SignalBase& sig, const ::py::function& callback, bool async) @@ -126,7 +119,7 @@ ::py::object signalDisconnectAll(SignalBase& sig, bool async) return resultObject(fut, async); } -::py::object proxySignalConnect(const AnyObjectGILSafe& obj, +::py::object proxySignalConnect(const AnyObject& obj, unsigned int signalId, const ::py::function& callback, bool async) @@ -135,18 +128,18 @@ ::py::object proxySignalConnect(const AnyObjectGILSafe& obj, return connect( [&](const SignalSubscriber& sub) { GILRelease unlock; - return obj->connect(signalId, sub).async(); + return obj.connect(signalId, sub).async(); }, callback, async); } -::py::object proxySignalDisconnect(const AnyObjectGILSafe& obj, +::py::object proxySignalDisconnect(const AnyObject& obj, SignalLink id, bool async) { const auto fut = [&] { GILRelease unlock; - return toFuture(obj->disconnect(id)); + return toFuture(obj.disconnect(id)); }(); GILAcquire lock; return resultObject(fut, async); @@ -219,8 +212,7 @@ void exportSignal(::py::module& m) arg(asyncArgName) = false) .def("disconnect", [](detail::ProxySignal& sig, SignalLink id, bool async) { - auto object = detail::AnyObjectGILSafe(sig.object); - return detail::proxySignalDisconnect(object, id, async); + return detail::proxySignalDisconnect(sig.object, id, async); }, "id"_a, arg(asyncArgName) = false) .def("__call__", @@ -228,8 +220,7 @@ void exportSignal(::py::module& m) const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); GILRelease unlock; - auto object = detail::AnyObjectGILSafe(sig.object); - object->metaPost(sig.signalId, args); + sig.object.metaPost(sig.signalId, args); }); } diff --git a/src/pytypes.cpp b/src/pytypes.cpp index ec2d617..088ba13 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -535,8 +535,10 @@ class StructuredIterableInterface res.reserve(_size); for (const ::py::handle itemHandle : obj) { - auto item = ::py::reinterpret_borrow<::py::object>(itemHandle); - res.push_back(AnyValue::from(item).release().rawValue()); + const auto item = ::py::reinterpret_borrow<::py::object>(itemHandle); + const auto itemRef = AnyValue::from(item).release(); + storeDisownedReference(storage, itemRef); + res.push_back(itemRef.rawValue()); } return res; } @@ -551,8 +553,10 @@ class StructuredIterableInterface // `pybind11::iterator` is not. We use advance instead. auto it = obj.begin(); std::advance(it, index); - const auto item = *it; - return AnyValue::from(item).release().rawValue(); + const auto item = ::py::reinterpret_borrow<::py::object>(*it); + const auto itemRef = AnyValue::from(item).release(); + storeDisownedReference(storage, itemRef); + return itemRef.rawValue(); } void set(void** /*storage*/, const std::vector&) override diff --git a/tests/moduletest.cpp b/tests/moduletest.cpp index b9e3108..717b773 100644 --- a/tests/moduletest.cpp +++ b/tests/moduletest.cpp @@ -121,6 +121,11 @@ class Cat return boost::make_shared(playCounter); } + void order(qi::AnyObject /*action*/) const + { + // Cats do not follow orders, they do nothing. + } + int nbPurr() { return purrCounter->load(); @@ -146,7 +151,8 @@ class Cat }; QI_REGISTER_OBJECT(Cat, meow, cloneMe, hunger, boredom, cuteness, - makePurr, makeSleep, makePlay, nbPurr, nbSleep, nbPlay); + makePurr, makeSleep, makePlay, order, + nbPurr, nbSleep, nbPlay); Cat::Cat() { From 1e1da980de17b2bdc15d253fd453072ad041b54c Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 9 Sep 2022 17:40:17 +0200 Subject: [PATCH 16/40] Upgrades libqi to version 3.0.0. --- cmake/set_libqi_dependency.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake index 31679dc..96182b0 100644 --- a/cmake/set_libqi_dependency.cmake +++ b/cmake/set_libqi_dependency.cmake @@ -1,4 +1,4 @@ -overridable_variable(LIBQI_VERSION 2.1.0) +overridable_variable(LIBQI_VERSION 3.0.0) # Our github clone is sometimes late or is missing tags, so we enable # customisation of the URL at configuration time, so users can use another clone. From b35f6dde07c81de7f5a8bb1c95eb76fb2f060423 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 9 Sep 2022 17:41:07 +0200 Subject: [PATCH 17/40] Bumps version to libqi-python 3.1.1 --- qi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qi/_version.py b/qi/_version.py index 7f5601d..726691b 100644 --- a/qi/_version.py +++ b/qi/_version.py @@ -1 +1 @@ -__version__ = '3.1.0' +__version__ = '3.1.1' From 43c3f546045bf075682312d3cab5e62bfe6b8027 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Mon, 10 Oct 2022 15:02:47 +0200 Subject: [PATCH 18/40] Enables build support of C++14. --- cmake/add_targets.cmake | 4 ++-- cmake/add_tests.cmake | 8 ++++---- cmake/set_dependencies.cmake | 8 ++++---- cmake/set_libqi_dependency.cmake | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmake/add_targets.cmake b/cmake/add_targets.cmake index cdb0ae1..0f448ab 100644 --- a/cmake/add_targets.cmake +++ b/cmake/add_targets.cmake @@ -46,7 +46,7 @@ enable_warnings(qi_python_objects) target_include_directories(qi_python_objects PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(qi_python_objects - PUBLIC cxx11 + PUBLIC cxx14 pybind11 qi.interface) @@ -90,7 +90,7 @@ target_sources(qi_python enable_warnings(qi_python) -target_link_libraries(qi_python PRIVATE cxx11 qi_python_objects) +target_link_libraries(qi_python PRIVATE cxx14 qi_python_objects) set_target_properties(qi_python PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME} diff --git a/cmake/add_tests.cmake b/cmake/add_tests.cmake index b69faf7..e4d2ef1 100644 --- a/cmake/add_tests.cmake +++ b/cmake/add_tests.cmake @@ -20,14 +20,14 @@ include(GoogleTest) find_package(qimodule REQUIRED HINTS ${libqi_SOURCE_DIR}) qi_create_module(moduletest NO_INSTALL) target_sources(moduletest PRIVATE tests/moduletest.cpp) -target_link_libraries(moduletest cxx11 qi.interface) +target_link_libraries(moduletest cxx14 qi.interface) enable_warnings(moduletest) set_build_rpath_to_qipython_dependencies(moduletest) add_executable(service_object_holder) target_sources(service_object_holder PRIVATE tests/service_object_holder.cpp) -target_link_libraries(service_object_holder PRIVATE cxx11 qi.interface) +target_link_libraries(service_object_holder PRIVATE cxx14 qi.interface) enable_warnings(service_object_holder) set_build_rpath_to_qipython_dependencies(service_object_holder) @@ -47,7 +47,7 @@ target_link_libraries(test_qipython pybind11 qi_python_objects qi.interface - cxx11 + cxx14 gmock) add_executable(test_qipython_local_interpreter) @@ -58,7 +58,7 @@ target_link_libraries(test_qipython_local_interpreter pybind11 qi_python_objects qi.interface - cxx11 + cxx14 gmock) set(_sdk_prefix "${CMAKE_BINARY_DIR}/sdk") diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake index ce237f0..5e354a8 100644 --- a/cmake/set_dependencies.cmake +++ b/cmake/set_dependencies.cmake @@ -153,9 +153,9 @@ if(CMAKE_TOOLCHAIN_FILE) endif() -# C++11 -add_library(cxx11 INTERFACE) -target_compile_features(cxx11 INTERFACE cxx_std_11) +# C++14 +add_library(cxx14 INTERFACE) +target_compile_features(cxx14 INTERFACE cxx_std_14) # Threads @@ -248,7 +248,7 @@ if(NOT pybind11_POPULATED) # subproject. add_library(pybind11 INTERFACE) target_include_directories(pybind11 INTERFACE ${pybind11_SOURCE_DIR}/include) - target_link_libraries(pybind11 INTERFACE cxx11 python_headers) + target_link_libraries(pybind11 INTERFACE cxx14 python_headers) endif() diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake index 96182b0..3a7acdc 100644 --- a/cmake/set_libqi_dependency.cmake +++ b/cmake/set_libqi_dependency.cmake @@ -40,7 +40,7 @@ else() include(set_libqi_dependency_system) endif() -target_link_libraries(qi.interface INTERFACE cxx11) +target_link_libraries(qi.interface INTERFACE cxx14) # Generate a Python file containing information about the native part of the # module.\ From 00653aeb9b95d84cec03e9380e74bf28bef98d35 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 1 Mar 2023 12:52:12 +0100 Subject: [PATCH 19/40] Upgrades libqi to version 4.0.0. --- cmake/set_libqi_dependency.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake index 3a7acdc..30ef957 100644 --- a/cmake/set_libqi_dependency.cmake +++ b/cmake/set_libqi_dependency.cmake @@ -1,4 +1,4 @@ -overridable_variable(LIBQI_VERSION 3.0.0) +overridable_variable(LIBQI_VERSION 4.0.0) # Our github clone is sometimes late or is missing tags, so we enable # customisation of the URL at configuration time, so users can use another clone. From f6877ebf7d24e2e5c0e11339e6ff7242d24432f1 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 1 Mar 2023 12:49:09 +0100 Subject: [PATCH 20/40] Upgrades supported C++ version to C++17. --- cmake/add_targets.cmake | 4 ++-- cmake/add_tests.cmake | 8 ++++---- cmake/set_dependencies.cmake | 8 ++++---- cmake/set_libqi_dependency.cmake | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmake/add_targets.cmake b/cmake/add_targets.cmake index 0f448ab..5f91d52 100644 --- a/cmake/add_targets.cmake +++ b/cmake/add_targets.cmake @@ -46,7 +46,7 @@ enable_warnings(qi_python_objects) target_include_directories(qi_python_objects PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(qi_python_objects - PUBLIC cxx14 + PUBLIC cxx17 pybind11 qi.interface) @@ -90,7 +90,7 @@ target_sources(qi_python enable_warnings(qi_python) -target_link_libraries(qi_python PRIVATE cxx14 qi_python_objects) +target_link_libraries(qi_python PRIVATE cxx17 qi_python_objects) set_target_properties(qi_python PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME} diff --git a/cmake/add_tests.cmake b/cmake/add_tests.cmake index e4d2ef1..8408498 100644 --- a/cmake/add_tests.cmake +++ b/cmake/add_tests.cmake @@ -20,14 +20,14 @@ include(GoogleTest) find_package(qimodule REQUIRED HINTS ${libqi_SOURCE_DIR}) qi_create_module(moduletest NO_INSTALL) target_sources(moduletest PRIVATE tests/moduletest.cpp) -target_link_libraries(moduletest cxx14 qi.interface) +target_link_libraries(moduletest cxx17 qi.interface) enable_warnings(moduletest) set_build_rpath_to_qipython_dependencies(moduletest) add_executable(service_object_holder) target_sources(service_object_holder PRIVATE tests/service_object_holder.cpp) -target_link_libraries(service_object_holder PRIVATE cxx14 qi.interface) +target_link_libraries(service_object_holder PRIVATE cxx17 qi.interface) enable_warnings(service_object_holder) set_build_rpath_to_qipython_dependencies(service_object_holder) @@ -47,7 +47,7 @@ target_link_libraries(test_qipython pybind11 qi_python_objects qi.interface - cxx14 + cxx17 gmock) add_executable(test_qipython_local_interpreter) @@ -58,7 +58,7 @@ target_link_libraries(test_qipython_local_interpreter pybind11 qi_python_objects qi.interface - cxx14 + cxx17 gmock) set(_sdk_prefix "${CMAKE_BINARY_DIR}/sdk") diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake index 5e354a8..46dc882 100644 --- a/cmake/set_dependencies.cmake +++ b/cmake/set_dependencies.cmake @@ -153,9 +153,9 @@ if(CMAKE_TOOLCHAIN_FILE) endif() -# C++14 -add_library(cxx14 INTERFACE) -target_compile_features(cxx14 INTERFACE cxx_std_14) +# C++17 +add_library(cxx17 INTERFACE) +target_compile_features(cxx17 INTERFACE cxx_std_17) # Threads @@ -248,7 +248,7 @@ if(NOT pybind11_POPULATED) # subproject. add_library(pybind11 INTERFACE) target_include_directories(pybind11 INTERFACE ${pybind11_SOURCE_DIR}/include) - target_link_libraries(pybind11 INTERFACE cxx14 python_headers) + target_link_libraries(pybind11 INTERFACE cxx17 python_headers) endif() diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake index 30ef957..ed2fa99 100644 --- a/cmake/set_libqi_dependency.cmake +++ b/cmake/set_libqi_dependency.cmake @@ -40,7 +40,7 @@ else() include(set_libqi_dependency_system) endif() -target_link_libraries(qi.interface INTERFACE cxx14) +target_link_libraries(qi.interface INTERFACE cxx17) # Generate a Python file containing information about the native part of the # module.\ From 6c5a30ea9824e13d4bbadbe7a9e1ea97622ceb70 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Thu, 2 Mar 2023 16:41:30 +0100 Subject: [PATCH 21/40] Bumps version to libqi-python 3.1.2. This release upgrades libqi to version 4.0.0 and sets the version of C++ to C++17. --- qi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qi/_version.py b/qi/_version.py index 726691b..f71b21a 100644 --- a/qi/_version.py +++ b/qi/_version.py @@ -1 +1 @@ -__version__ = '3.1.1' +__version__ = '3.1.2' From 711a06163c8bc0eeaa8aad513aa05401d64d9e72 Mon Sep 17 00:00:00 2001 From: Vincent PALANCHER Date: Fri, 9 Jun 2023 18:17:23 +0000 Subject: [PATCH 22/40] build: Rewrites project build definition. #SW-159 This patch rewrites the definition of the build of libqi-python in order to remove the dependency to `qibuild` and `qi toolchains` and instead use more standard tools and conventions. It completely refactors the `CMakeLists.txt` files to use an approach closer to what is referred to as "modern CMake". The project now requires CMake 3.23 or greater. The project defines a single install components, the `Module` component. Tests are now declared to CMake using `add_test` and the CTest module is used. This means that tests can now be executed with the `ctest` command line tool. This patch also adds the support of Conan, to download the project dependencies and to create a Conan package of the project. Please refer to the README for basic usage. It also rewrites the Python module project definition using PEP517 and `scikit-build-core` as a build backend. --- .gitignore | 3 +- CMakeLists.txt | 430 ++++++++++++++++-- MANIFEST.in | 6 + README.rst | 234 +++------- cmake/BuildType.cmake | 16 + .../{utils.cmake => ParsePythonVersion.cmake} | 58 --- cmake/add_targets.cmake | 105 ----- cmake/add_tests.cmake | 124 ----- cmake/install_runtime_dependencies.cmake | 117 ----- cmake/native.py.in | 1 - cmake/set_dependencies.cmake | 306 ------------- cmake/set_globals.cmake | 50 -- cmake/set_install_rules.cmake | 7 - cmake/set_install_rules_standalone.cmake | 57 --- cmake/set_install_rules_system.cmake | 35 -- cmake/set_libqi_dependency.cmake | 53 --- cmake/set_libqi_dependency_standalone.cmake | 71 --- cmake/set_libqi_dependency_system.cmake | 42 -- conanfile.py | 97 ++++ pyproject.toml | 60 ++- qi/__init__.py | 8 - qi/_version.py | 1 - qi/_version.py.in | 1 + qi/native.py.in | 1 + qiproject.xml | 13 - qipython.pml | 4 - setup.py | 67 --- 27 files changed, 635 insertions(+), 1332 deletions(-) create mode 100644 cmake/BuildType.cmake rename cmake/{utils.cmake => ParsePythonVersion.cmake} (50%) delete mode 100644 cmake/add_targets.cmake delete mode 100644 cmake/add_tests.cmake delete mode 100644 cmake/install_runtime_dependencies.cmake delete mode 100644 cmake/native.py.in delete mode 100644 cmake/set_dependencies.cmake delete mode 100644 cmake/set_globals.cmake delete mode 100644 cmake/set_install_rules.cmake delete mode 100644 cmake/set_install_rules_standalone.cmake delete mode 100644 cmake/set_install_rules_system.cmake delete mode 100644 cmake/set_libqi_dependency.cmake delete mode 100644 cmake/set_libqi_dependency_standalone.cmake delete mode 100644 cmake/set_libqi_dependency_system.cmake create mode 100644 conanfile.py delete mode 100644 qi/_version.py create mode 100644 qi/_version.py.in create mode 100644 qi/native.py.in delete mode 100644 qiproject.xml delete mode 100644 qipython.pml delete mode 100755 setup.py diff --git a/.gitignore b/.gitignore index cb55908..136d222 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ build* _skbuild dist -*.egg-info \ No newline at end of file +*.egg-info +CMakeUserPresets.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f3efc..b2a5f22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,39 +1,405 @@ -cmake_minimum_required(VERSION 3.16) +## Copyright (c) 2023 Aldebaran Robotics. All rights reserved. +## Use of this source code is governed by a BSD-style license that can be +## found in the COPYING file. +## +## qi_python CMake project +## ======================= +## +## Parameters: +## ----------- +## BUILD_TESTING +## ~~~~~~~~~~~~~ +## If set to true, enables building the tests. See the documentation of the +## `CTest` module for more details on this variable. + +cmake_minimum_required(VERSION 3.23) if(ANDROID) message(FATAL_ERROR "This project does not support Android, stopping.") endif() -# Find our project CMake files in the dedicated subdirectory. -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - -include(utils) -include(set_globals) +include(cmake/ParsePythonVersion.cmake) -# Read the version of the library directly from the python file and sets it +# Read the version of the library directly from the `pyproject.toml` file and sets it # as the version of the project. -# -# The file is expected to contain a line `__version__ = `. -set(_version_file ${CMAKE_CURRENT_SOURCE_DIR}/qi/_version.py) -file(STRINGS ${_version_file} _version_strings REGEX version) -set(_version_regex "__version__[ \t]*=[ \t]*['\"]([^'\"]+)['\"]") -if(NOT _version_strings MATCHES "${_version_regex}") - message(FATAL_ERROR "Could not find the package version in file '${_version_file}'.") -endif() -parse_python_version(_version ${CMAKE_MATCH_1}) - -project(libqi-python VERSION ${_version_VERSION_RELEASE}) - -option(QIPYTHON_STANDALONE - "If set, builds and installs the module as a standalone, ready to be \ -packaged as a wheel. Otherwise, builds and installs it, ready to be included \ -as part of a system (such as NAOqi) that contains its dependencies." - FALSE) - -# Including this package creates the common options/flags/functions of qi projects. -find_package(qibuild) - -include(set_dependencies) -include(add_targets) -include(set_install_rules) -include(add_tests) +set(pyproject_file "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml") +file(STRINGS ${pyproject_file} version_strings REGEX version) +set(version_regex "version[ \t]*=[ \t]*\"([^\"]+)\"") +if(NOT version_strings MATCHES "${version_regex}") + message(FATAL_ERROR "Could not find the package version in file '${pyproject_file}'.") +endif() +parse_python_version(QI_PYTHON "${CMAKE_MATCH_1}") + +project(qi_python VERSION "${QI_PYTHON_VERSION_RELEASE}") + +include(cmake/BuildType.cmake) + +# Enable testing with CTest. This defines the BUILD_TESTING option. +# Disable tests by default when cross compiling. +set(build_testing_default TRUE) +if(CMAKE_CROSSCOMPILING) + set(build_testing_default FALSE) +endif() +option(BUILD_TESTING "Build the testing tree." "${build_testing_default}") +include(CTest) + +if(MSVC) + add_compile_options(/W3) +else() + add_compile_options(-Wall -Wextra) +endif() + +# Output everything directly in predefined directories of the build tree. +# This is required by the SDK layout implementation. +# Also write a "path.conf" file, which is required for executing some tests. +set(sdk_dir "${CMAKE_BINARY_DIR}/sdk") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${sdk_dir}/bin") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${sdk_dir}/lib") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${sdk_dir}/lib") +set(path_conf_dir "${sdk_dir}/share/qi") +set(path_conf_file_path "${path_conf_dir}/path.conf") +if(NOT EXISTS "${path_conf_file_path}") + file(MAKE_DIRECTORY "${path_conf_dir}") + file(TOUCH "${path_conf_file_path}") +endif() + +############################################################################## +# Find Python +############################################################################## +# First search for Python with Interpreter+Devel components, to allow the module +# to look for the interpreter that goes with the Python development library. +# Then if it could not find both, try looking for the development component +# only, but this time force the module to find it or fail (REQUIRED). +# Some of our toolchains (when crosscompiling for instance) have the Python +# library but not an interpreter that can run on the host. +find_package(Python COMPONENTS Development Interpreter) +if(NOT Python_FOUND) + find_package(Python REQUIRED COMPONENTS Development) +endif() + +############################################################################## +# Find Boost +############################################################################## +find_package( + Boost REQUIRED + COMPONENTS thread +) + +############################################################################## +# Find Pybind11 +############################################################################## +find_package(pybind11 REQUIRED) + +############################################################################## +# Find LibQi +############################################################################## +find_package(qi REQUIRED) + +############################################################################## +# Convenience library: cxx_standard +############################################################################## +add_library(cxx_standard INTERFACE) + +# The project requires at least C++17. +target_compile_features( + cxx_standard + INTERFACE + cxx_std_17 +) + +############################################################################## +# Library: qi_python_objects +############################################################################## +add_library(qi_python_objects OBJECT) + +target_sources( + qi_python_objects + + PUBLIC + qipython/common.hpp + qipython/pyapplication.hpp + qipython/pyasync.hpp + qipython/pyclock.hpp + qipython/pyexport.hpp + qipython/pyfuture.hpp + qipython/pylog.hpp + qipython/pymodule.hpp + qipython/pyobject.hpp + qipython/pypath.hpp + qipython/pyproperty.hpp + qipython/pysession.hpp + qipython/pysignal.hpp + qipython/pyguard.hpp + qipython/pytranslator.hpp + qipython/pytypes.hpp + qipython/pystrand.hpp + + PRIVATE + src/pyapplication.cpp + src/pyasync.cpp + src/pyclock.cpp + src/pyexport.cpp + src/pyfuture.cpp + src/pylog.cpp + src/pymodule.cpp + src/pyobject.cpp + src/pypath.cpp + src/pyproperty.cpp + src/pysession.cpp + src/pysignal.cpp + src/pystrand.cpp + src/pytranslator.cpp + src/pytypes.cpp +) + +target_include_directories( + qi_python_objects + PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" +) + +target_link_libraries( + qi_python_objects + PUBLIC + cxx_standard + Boost::headers + Boost::thread + pybind11::pybind11 + qi::qi +) + +set_target_properties( + qi_python_objects + PROPERTIES + CXX_EXTENSIONS TRUE + CXX_VISIBILITY_PRESET hidden + # Use PIC as the library is linked into a shared library in the qi_python + # target. + POSITION_INDEPENDENT_CODE TRUE +) + +############################################################################## +# Library: qi_python +############################################################################## +pybind11_add_module( + qi_python + # Disable generation of an extension for the shared library. + WITHOUT_SOABI +) + +# Generate a Python file containing the version. +set(version_py_file "${CMAKE_CURRENT_BINARY_DIR}/qi/_version.py") +configure_file(qi/_version.py.in "${version_py_file}" @ONLY) + +# Generate a Python file containing information about the native part of the +# module. +set(native_py_file "${CMAKE_CURRENT_BINARY_DIR}/qi/native.py") +if(qi_VERSION) + set(NATIVE_VERSION "${qi_VERSION}") +else() + set(NATIVE_VERSION "unknown") +endif() +configure_file(qi/native.py.in "${native_py_file}" @ONLY) + +set( + qi_python_py_files + qi/__init__.py + qi/logging.py + qi/path.py + qi/translator.py + qi/_binder.py + qi/_type.py +) +set( + qi_python_test_py_files + qi/test/__init__.py + qi/test/conftest.py + qi/test/test_applicationsession.py + qi/test/test_async.py + qi/test/test_call.py + qi/test/test_log.py + qi/test/test_module.py + qi/test/test_promise.py + qi/test/test_property.py + qi/test/test_return_empty_object.py + qi/test/test_session.py + qi/test/test_signal.py + qi/test/test_strand.py + qi/test/test_typespassing.py +) + +target_sources( + qi_python + PUBLIC + ${qi_python_py_files} + PRIVATE + src/module.cpp + ${qi_python_test_py_files} +) + +target_link_libraries( + qi_python + PRIVATE + cxx_standard + qi_python_objects +) + +set_target_properties( + qi_python + PROPERTIES + CXX_EXTENSIONS TRUE + CXX_VISIBILITY_PRESET hidden + LIBRARY_OUTPUT_DIRECTORY qi + RUNTIME_OUTPUT_DIRECTORY qi + INSTALL_RPATH_USE_LINK_PATH TRUE +) + +############################################################################## +# Library: qipython_module_plugin +############################################################################## +add_library(qimodule_python_plugin SHARED) +target_sources( + qimodule_python_plugin + PRIVATE + src/qimodule_python_plugin.cpp +) +target_link_libraries( + qimodule_python_plugin + PRIVATE + cxx_standard + qi::qi +) + +############################################################################## +# Installation +############################################################################## +install( + TARGETS qi_python + LIBRARY DESTINATION qi COMPONENT Module + # On Windows, shared libraries (DLL) are considered `RUNTIME`. + RUNTIME DESTINATION qi COMPONENT Module +) + +# Install Python files. +install( + FILES ${qi_python_py_files} "${version_py_file}" "${native_py_file}" + DESTINATION qi + COMPONENT Module +) + +############################################################################## +# Tests +############################################################################## +if(BUILD_TESTING) + find_package(GTest REQUIRED) + + # Use the CMake GoogleTest module that offers functions to automatically + # discover tests. + include(GoogleTest) + + find_package(qimodule REQUIRED) + qi_create_module(moduletest NO_INSTALL) + target_sources( + moduletest + PRIVATE + tests/moduletest.cpp + ) + target_link_libraries( + moduletest + PRIVATE + cxx_standard + ) + + add_executable(service_object_holder) + target_sources( + service_object_holder + PRIVATE + tests/service_object_holder.cpp + ) + target_link_libraries( + service_object_holder + PRIVATE + cxx_standard + qi::qi + ) + + add_executable(test_qipython) + target_sources( + test_qipython + PRIVATE + tests/common.hpp + tests/test_qipython.cpp + tests/test_guard.cpp + tests/test_types.cpp + tests/test_signal.cpp + tests/test_property.cpp + tests/test_object.cpp + tests/test_module.cpp + ) + target_link_libraries( + test_qipython + PRIVATE + qi_python_objects + cxx_standard + Python::Python + Boost::headers + pybind11::pybind11 + GTest::gmock + ) + gtest_discover_tests(test_qipython) + + add_executable(test_qipython_local_interpreter) + target_sources( + test_qipython_local_interpreter + PRIVATE + tests/test_qipython_local_interpreter.cpp + ) + target_link_libraries( + test_qipython_local_interpreter + PRIVATE + Python::Python + pybind11::pybind11 + qi_python_objects + cxx_standard + GTest::gmock + ) + gtest_discover_tests(test_qipython_local_interpreter) + + if(NOT Python_Interpreter_FOUND) + message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") + else() + message(STATUS "tests: a compatible Python Interpreter was found, Python tests are enabled.") + + execute_process( + COMMAND "${Python_EXECUTABLE}" -m pytest --version + OUTPUT_QUIET + ERROR_QUIET + RESULT_VARIABLE pytest_version_err + ) + if(pytest_version_err) + message(WARNING "tests: the pytest module does not seem to be installed for this Python Interpreter, " + "the execution of the tests will most likely result in a failure.") + endif() + + # Copy python files so that tests can be run in the build directory, with the generated files. + foreach(file IN LISTS qi_python_py_files qi_python_test_py_files) + configure_file("${file}" "${file}" COPYONLY) + endforeach() + + add_test( + NAME pytest + COMMAND + Python::Interpreter + -m pytest + "${CMAKE_CURRENT_BINARY_DIR}/qi/test" + --exec_path "$ --qi-standalone --qi-listen-url tcp://127.0.0.1:0" + ) + set_tests_properties( + pytest + PROPERTIES + ENVIRONMENT "QI_SDK_PREFIX=${sdk_dir}" + ) + endif() +else() + message(STATUS "tests: tests are disabled") +endif() diff --git a/MANIFEST.in b/MANIFEST.in index 9491f80..cff072a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,8 @@ include README.rst include COPYING +include CMakeLists.txt +graft cmake +graft qi +graft examples +graft qipython +graft src diff --git a/README.rst b/README.rst index 1b65663..2383130 100644 --- a/README.rst +++ b/README.rst @@ -10,193 +10,113 @@ __ LibQi_repo_ Building ======== -This project supports the building of a *standalone* package (for instance as a -wheel that can be uploaded on PyPi_) or of a "system" archive. +The libqi-python project requires a compiler that supports C++17 to build. -.. _standalone: +It is built with CMake >= 3.23. -Standalone (wheel) ------------------- +.. note:: + The CMake project offers several configuration options and exports a set + of targets when installed. You may refer to the ``CMakeLists.txt`` file + for more details about available parameters and exported targets. -This build mode is also referred to as the *standalone* mode. It is enabled by -passing ``-DQIPYTHON_STANDALONE=ON`` to the CMake call. The Python setup script -also sets this mode automatically when used. +.. note:: + The procedures described below assume that you have downloaded the project + sources and that your current working directory is the project sources root + directory. -In this mode, the project will build libqi and install all its dependencies as -part of the project. +Conan +^^^^^ -The package can be built from the ``setup.py`` script: +Additionally, libqi-python is available as a Conan 2 project, which means you +can use Conan to fetch dependencies. -.. code:: bash +You can install and/or upgrade Conan 2 and create a default profile in the +following way: - python3 ./setup.py bdist_wheel +.. code-block:: console -or + # install/upgrade Conan 2 + pip install --upgrade conan~=2 + # create a default profile + conan profile detect -.. code:: bash +Install dependencies from Conan and build with CMake +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - pip3 wheel . --no-use-pep517 +The procedure to build the project using Conan to fetch dependencies is the +following. -The use of PEP517 is not yet supported and still problematic with our setup -script, so we have to disable it. +You must first install the project dependencies in Conan. -The setup script uses scikit-build_ which is itself based on setuptools_. It -handles any option the latter can handle. Additionally, it can take CMake -arguments, which means that you can almost entirely customize how the native -part is built. +.. code-block:: console -In particular, you can use the ``CMAKE_TOOLCHAIN_FILE`` variable to specify a -toolchain to build the native part of the wheel (e.g. if you are using qi -toolchains): + conan install . --build=missing -s build_type=Debug -.. code:: bash +This will generate a build directory containing a configuration with a +toolchain file that allows CMake to find dependencies inside the Conan cache. - python3 ./setup.py bdist_wheel -DCMAKE_TOOLCHAIN_FILE=$HOME/.local/share/qi/toolchains/my_toolchain/toolchain-my_toolchain.cmake +You can then invoke CMake directly inside the build configuration directory to +configure and build the project. Fortunately, Conan also generates a CMake +preset that simplifies the process. The name of the preset may differ on +your machine. You may need to find the preset generated by Conan first by +calling: -System archive --------------- - -This build mode is also referred to as the "system" mode. This is the default -mode. - -In this mode, CMake will expect libqi to be available as a binary package. The -simplest way to build the package in this mode is to use qibuild, which will -first build libqi then libqi-python. - -.. code:: bash - - qibuild configure - qibuild make - -You can also set the ``QI_DIR`` variable at the CMake call to let it know it of -the location of the libqi package. - -.. code:: bash - - mkdir build && cd build - cmake .. -DQI_DIR=/path/to/libqi/install/dir - cmake --build . - - -Technical details ------------------ - -The compiled/native part of this project is based on CMake. It can be built as -any other project of the sort and also through qibuild_, although it requires at -least CMake v3.17. - -Our CMake scripts may take a few parameters: - -- ``QIPYTHON_STANDALONE``, when set, builds the library in *standalone* mode. - Refer to the standalone_ section for details. -- ``QIPYTHON_FORCE_STRIP``, when set, forces the build system to strip the - libqi-python native module library at install, resulting in a smaller binary. -- ``QI_WITH_TESTS``, when set, enables building of tests. This option is ignored - when cross-compiling. +.. code-block:: console -Dependencies -~~~~~~~~~~~~ + cmake --list-presets -The project has a few dependencies and the build system might report errors if -it fails to find them. It uses either ``FindXXX`` CMake modules through the -``find_package`` command or the ``FetchContent`` module for subprojects. Either way, -any parameter that these modules and the ``find_package`` command accept can be -used to customize how the build system finds the libraries or fetches the -subprojects. +Here, we'll assume that the preset is named `conan-linux-x86_64-gcc-debug`. +To start building, you need to configure with CMake and then build: -Most of the variables described here are defined in the -``cmake/set_dependencies.cmake`` file. You may refer to this file for more details -on these variables and their values. +.. code-block:: console -LibQi ->>>>> + cmake --preset conan-linux-x86_64-gcc-debug + cmake --build --preset conan-linux-x86_64-gcc-debug -The project's dependencies on LibQi depends on the building mode: +You can then invoke tests using CTest_: -- In **system mode**, it will expect to find it as a binary package. The location - of the binary package installation can be specified through the ``QI_DIR`` - variable. -- In **standalone mode**, it will download and compile it as a subproject through - the ``FetchContent`` CMake module. How it is downloaded can be customized - through the following variables: +.. code-block:: console - - ``LIBQI_VERSION`` - - ``LIBQI_GIT_REPOSITORY`` - - ``LIBQI_GIT_TAG`` + ctest --preset conan-linux-x86_64-gcc-debug --output-on-failure -It is possible to skip the download step and use an existing source directory by -setting its path as the ``FETCHCONTENT_SOURCE_DIR_LIBQI`` CMake variable. The -build system will still check that the version of the sources matches the -``LIBQI_VERSION`` value if it is set. +Finally, you can install the project in the directory of your choice. -Python ->>>>>> +The project defines a single install component, the ``Module`` component. -The build system uses the FindPython_ CMake module. It will try to honor the -following variables if they are set: +.. code-block:: console - - ``PYTHON_VERSION_STRING`` - - ``PYTHON_LIBRARY`` - - ``PYTHON_INCLUDE_DIR`` + # "cmake --install" does not support preset sadly. + cmake --install build/linux-x86_64-gcc-debug + --component Module --prefix ~/my-libqi-python-install -pybind11 ->>>>>>>> +Wheel (PEP 517) +--------------- -The build system will by default download and compile pybind11__ as a -subproject through the ``FetchContent`` CMake module. How it is downloaded can be -customized through the following variables: +You may build this project as a wheel package using PEP 517. -__ pybind11_repo_ +It uses a scikit-build_ backend which interfaces with CMake. - - ``PYBIND11_VERSION`` - - ``PYBIND11_GIT_REPOSITORY`` - - ``PYBIND11_GIT_TAG`` +You may need to provide a toolchain file so that CMake finds the required +dependencies, such as a toolchain generated by Conan: +.. code-block:: console -Boost ->>>>> + conan install . --build=missing -The build system will look for the Boost libraries on the system or in the -toolchain if one is set. The expected version of the libraries is specified as -the ``BOOST_VERSION`` variable. +You now can use the ``build`` Python module to build the wheel using PEP 517. -The build system uses the FindBoost_ CMake module. +.. code-block:: console -OpenSSL ->>>>>>> + export CMAKE_TOOLCHAIN_FILE=$PWD/build/linux-x86_64-gcc-release/generators/conan_toolchain.cmake + python -m build -The build system uses the FindOpenSSL_ CMake module. +When built that way, the native libraries present in the wheel are most likely incomplete. +You will need to use ``auditwheel`` or ``delocate`` to fix it. -ICU ->>> - -The build system uses the FindICU_ CMake module. - -GoogleTest ->>>>>>>>>> - -The build system will by default download and compile GoogleTest__ as a -subproject through the ``FetchContent`` CMake module. How it is downloaded can be -customized through the following variables: - -__ GoogleTest_repo_ - - - ``GOOGLETEST_VERSION`` - - ``GOOGLETEST_GIT_REPOSITORY`` - - ``GOOGLETEST_GIT_TAG`` - -Install -~~~~~~~ - -Once the project is configured, it can be built and installed as any CMake -project: - -.. code:: bash - - mkdir build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/myinstallpath - cmake --install . +.. code-block:: console + auditwheel repair --plat manylinux_2_31_x86_64 dist/qi-*.whl + # The wheel will be by default placed in a `./wheelhouse/` directory. Crosscompiling -------------- @@ -207,26 +127,8 @@ path of the CMake file in your toolchain. __ CMake_toolchains_ -Testing -======= - -When enabled, tests can be executed with `CTest`_. - -.. code:: bash - - cd build - ctest . --output-on-failure - .. _LibQi_repo: https://github.com/aldebaran/libqi -.. _PyPi: https://pypi.org/ .. _scikit-build: https://scikit-build.readthedocs.io/en/latest/ .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html -.. _qibuild: https://github.com/aldebaran/qibuild -.. _pybind11_repo: https://pybind11.readthedocs.io/en/latest/ -.. _FindPython: https://cmake.org/cmake/help/latest/module/FindPython.html -.. _FindBoost: https://cmake.org/cmake/help/latest/module/FindBoost.html -.. _FindOpenSSL: https://cmake.org/cmake/help/latest/module/FindOpenSSL.html -.. _FindICU: https://cmake.org/cmake/help/latest/module/FindICU.html -.. _GoogleTest_repo: https://github.com/google/googletest .. _CMake_toolchains: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html .. _CTest: https://cmake.org/cmake/help/latest/manual/ctest.1.html diff --git a/cmake/BuildType.cmake b/cmake/BuildType.cmake new file mode 100644 index 0000000..7e31d51 --- /dev/null +++ b/cmake/BuildType.cmake @@ -0,0 +1,16 @@ +# Inspired by https://www.kitware.com/cmake-and-the-default-build-type/. +# +# Set a default build type if none was specified. +set(default_build_type "Release") +if(EXISTS "${CMAKE_SOURCE_DIR}/.git") + set(default_build_type "Debug") +endif() + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to '${default_build_type}' as none was specified.") + set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE + STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() diff --git a/cmake/utils.cmake b/cmake/ParsePythonVersion.cmake similarity index 50% rename from cmake/utils.cmake rename to cmake/ParsePythonVersion.cmake index 5db32ab..4f38d1a 100644 --- a/cmake/utils.cmake +++ b/cmake/ParsePythonVersion.cmake @@ -1,5 +1,3 @@ -include_guard(GLOBAL) - # Parses a Python version as defined in PEP440 # (see https://www.python.org/dev/peps/pep-0440/) in some input and sets the # following variables accordingly: @@ -48,59 +46,3 @@ function(parse_python_version prefix input) set(${prefix}_VERSION_RELEASE_MAJOR_MINOR "${CMAKE_MATCH_1}${CMAKE_MATCH_2}" PARENT_SCOPE) set(${prefix}_VERSION_RELEASE_PATCH "${CMAKE_MATCH_5}" PARENT_SCOPE) endfunction() - -# Enables most compiler warnings for a target. -function(enable_warnings target) - if(CMAKE_COMPILER_IS_GNUCC) - target_compile_options(${target} PRIVATE -Wall -Wextra) - elseif(MSVC) - target_compile_options(${target} PRIVATE /W4) - endif() -endfunction() - -# Sets the install RPATH of some targets to a list of paths relative to the -# location of the binary. -function(set_install_rpath) - cmake_parse_arguments(SET_INSTALL_RPATH "ORIGIN" "" "TARGETS;PATHS_REL_TO_ORIGIN" ${ARGN}) - foreach(_target IN LISTS SET_INSTALL_RPATH_TARGETS) - set(_paths) - if(SET_INSTALL_RPATH_ORIGIN) - message(VERBOSE "Adding $ORIGIN to RPATH of ${_target}") - list(APPEND _paths - "$<$:$ORIGIN>$<$:@loader_path>") - endif() - foreach(_path IN LISTS SET_INSTALL_RPATH_PATHS_REL_TO_ORIGIN) - if(NOT _path) - continue() - endif() - message(VERBOSE "Adding $ORIGIN/${_path} to RPATH of ${_target}") - list(APPEND _paths - "$<$:$ORIGIN/${_path}>$<$:@loader_path/${_path}>") - endforeach() - if(_paths) - message(VERBOSE "Setting ${_target} RPATH to ${_paths}") - set_property(TARGET ${_target} PROPERTY INSTALL_RPATH ${_paths}) - endif() - endforeach() -endfunction() - -# Creates a variable that can be overridden by the user from either the -# command-line, the cache or the environment. -# The order of preference is: -# - the value from the variable in cache (or the command-line, since setting -# a variable from the command-line automatically adds it to the cache). -# - the value from the variable in the environment. -# - the default value for the variable. -function(overridable_variable name default_value) - # The variable already exists in the cache. It's the preferred value, so we - # don't change it. - if(DEFINED CACHE{${name}}) - return() - endif() - - set(value ${default_value}) - if(DEFINED ENV{${name}}) - set(value $ENV{${name}}) - endif() - set(${name} ${value} PARENT_SCOPE) -endfunction() diff --git a/cmake/add_targets.cmake b/cmake/add_targets.cmake deleted file mode 100644 index 5f91d52..0000000 --- a/cmake/add_targets.cmake +++ /dev/null @@ -1,105 +0,0 @@ -include_guard(GLOBAL) - - -add_library(qi_python_objects OBJECT) - -target_sources(qi_python_objects - PUBLIC - qipython/common.hpp - qipython/pyapplication.hpp - qipython/pyasync.hpp - qipython/pyclock.hpp - qipython/pyexport.hpp - qipython/pyfuture.hpp - qipython/pylog.hpp - qipython/pymodule.hpp - qipython/pyobject.hpp - qipython/pypath.hpp - qipython/pyproperty.hpp - qipython/pysession.hpp - qipython/pysignal.hpp - qipython/pyguard.hpp - qipython/pytranslator.hpp - qipython/pytypes.hpp - qipython/pystrand.hpp - - PRIVATE - src/pyapplication.cpp - src/pyasync.cpp - src/pyclock.cpp - src/pyexport.cpp - src/pyfuture.cpp - src/pylog.cpp - src/pymodule.cpp - src/pyobject.cpp - src/pypath.cpp - src/pyproperty.cpp - src/pysession.cpp - src/pysignal.cpp - src/pystrand.cpp - src/pytranslator.cpp - src/pytypes.cpp -) - -enable_warnings(qi_python_objects) - -target_include_directories(qi_python_objects PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - -target_link_libraries(qi_python_objects - PUBLIC cxx17 - pybind11 - qi.interface) - -set_target_properties(qi_python_objects - # Use PIC as the library is linked into a shared library in the qi_python - # target. - PROPERTIES POSITION_INDEPENDENT_CODE TRUE) - - -Python_add_library(qi_python MODULE) - -set(QIPYTHON_PYTHON_MODULE_FILES - qi/__init__.py - qi/logging.py - qi/path.py - qi/translator.py - qi/_binder.py - qi/_type.py - qi/_version.py) - -target_sources(qi_python - PRIVATE src/module.cpp - ${QIPYTHON_PYTHON_MODULE_FILES} - qi/test/__init__.py - qi/test/conftest.py - qi/test/test_applicationsession.py - qi/test/test_async.py - qi/test/test_call.py - qi/test/test_log.py - qi/test/test_module.py - qi/test/test_promise.py - qi/test/test_property.py - qi/test/test_return_empty_object.py - qi/test/test_session.py - qi/test/test_signal.py - qi/test/test_strand.py - qi/test/test_typespassing.py - setup.py - pyproject.toml - README.rst) - -enable_warnings(qi_python) - -target_link_libraries(qi_python PRIVATE cxx17 qi_python_objects) - -set_target_properties(qi_python - PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME} - RUNTIME_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME}) - -set_build_rpath_to_qipython_dependencies(qi_python) - -if(NOT QIPYTHON_STANDALONE) - add_library(qimodule_python_plugin SHARED) - target_sources(qimodule_python_plugin PRIVATE src/qimodule_python_plugin.cpp) - target_link_libraries(qimodule_python_plugin PRIVATE qi.interface) -endif() diff --git a/cmake/add_tests.cmake b/cmake/add_tests.cmake deleted file mode 100644 index 8408498..0000000 --- a/cmake/add_tests.cmake +++ /dev/null @@ -1,124 +0,0 @@ -include_guard(GLOBAL) - -if(NOT QI_WITH_TESTS) - message(STATUS "tests: tests are disabled") - return() -endif() - -if(CMAKE_CROSSCOMPILING) - message(STATUS "tests: crosscompiling, tests are disabled") - return() -endif() - -enable_testing() - -# Use the CMake GoogleTest module that offers functions to automatically discover -# tests. -include(GoogleTest) - - -find_package(qimodule REQUIRED HINTS ${libqi_SOURCE_DIR}) -qi_create_module(moduletest NO_INSTALL) -target_sources(moduletest PRIVATE tests/moduletest.cpp) -target_link_libraries(moduletest cxx17 qi.interface) -enable_warnings(moduletest) -set_build_rpath_to_qipython_dependencies(moduletest) - - -add_executable(service_object_holder) -target_sources(service_object_holder PRIVATE tests/service_object_holder.cpp) -target_link_libraries(service_object_holder PRIVATE cxx17 qi.interface) -enable_warnings(service_object_holder) -set_build_rpath_to_qipython_dependencies(service_object_holder) - - -add_executable(test_qipython) -target_sources(test_qipython - PRIVATE tests/common.hpp - tests/test_qipython.cpp - tests/test_guard.cpp - tests/test_types.cpp - tests/test_signal.cpp - tests/test_property.cpp - tests/test_object.cpp - tests/test_module.cpp) -target_link_libraries(test_qipython - PRIVATE Python::Python - pybind11 - qi_python_objects - qi.interface - cxx17 - gmock) - -add_executable(test_qipython_local_interpreter) -target_sources(test_qipython_local_interpreter - PRIVATE tests/test_qipython_local_interpreter.cpp) -target_link_libraries(test_qipython_local_interpreter - PRIVATE Python::Python - pybind11 - qi_python_objects - qi.interface - cxx17 - gmock) - -set(_sdk_prefix "${CMAKE_BINARY_DIR}/sdk") -foreach(test_target IN ITEMS test_qipython - test_qipython_local_interpreter) - # Unfortunately, in some of our toolchains, gtest/gmock headers are found in the qi-framework - # package, which comes first in the include paths order given to the compiler. This causes the - # compiler to use those headers instead of the ones we got from a `FetchContent` of the googletest - # repository. - target_include_directories(${test_target} - BEFORE # Force this path to come first in the list of include paths. - PRIVATE $) - enable_warnings(${test_target}) - set_build_rpath_to_qipython_dependencies(${test_target}) - gtest_discover_tests(${test_target} EXTRA_ARGS --qi-sdk-prefix ${_sdk_prefix}) -endforeach() - -if(NOT Python_Interpreter_FOUND) - message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") -else() - message(STATUS "tests: a compatible Python Interpreter was found, Python tests are enabled.") - - # qibuild sets these variables which tends to mess up our calls of the Python interpreter, so - # we reset them preemptively. - set(ENV{PYTHONHOME}) - set(ENV{PYTHONPATH}) - - execute_process(COMMAND "${Python_EXECUTABLE}" -m pytest --version - OUTPUT_QUIET ERROR_QUIET - RESULT_VARIABLE pytest_version_err) - if(pytest_version_err) - message(WARNING "tests: the pytest module does not seem to be installed for this Python Interpreter\ -, the execution of the tests will most likely result in a failure.") - endif() - - macro(copy_py_files dir) - file(GLOB _files RELATIVE "${PROJECT_SOURCE_DIR}" CONFIGURE_DEPENDS "${dir}/*.py") - foreach(_file IN LISTS _files) - set_property(DIRECTORY APPEND PROPERTY - CMAKE_CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/${_file}") - file(COPY "${_file}" DESTINATION "${dir}") - endforeach() - endmacro() - - copy_py_files(qi) - copy_py_files(qi/test) - - add_test(NAME pytest - COMMAND Python::Interpreter -m pytest qi/test - --exec_path - "$ --qi-standalone --qi-listen-url tcp://127.0.0.1:0") - - get_filename_component(_ssl_dir "${OPENSSL_SSL_LIBRARY}" DIRECTORY) - set(_pytest_env - "QI_SDK_PREFIX=${_sdk_prefix}" - "LD_LIBRARY_PATH=${_ssl_dir}") - set_tests_properties(pytest PROPERTIES ENVIRONMENT "${_pytest_env}") -endif() - -# Ensure compatibility with qitest by simply running ctest. Timeout is set to 4 minutes. -file(WRITE - "${CMAKE_BINARY_DIR}/qitest.cmake" "--name;run_ctest;--timeout;240;--working-directory;\ -${CMAKE_CURRENT_BINARY_DIR};--env;PYTHONHOME=;--env;PYTHONPATH=;--;${CMAKE_CTEST_COMMAND};-V") diff --git a/cmake/install_runtime_dependencies.cmake b/cmake/install_runtime_dependencies.cmake deleted file mode 100644 index ae6f787..0000000 --- a/cmake/install_runtime_dependencies.cmake +++ /dev/null @@ -1,117 +0,0 @@ -set(_module_install_dir - "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/${QIPYTHON_PYTHON_MODULE_NAME}") - -# On Windows, shared libraries (DLL) are not in the `lib` subdirectory of -# the dependencies, but in the `bin` directory. So for each dependency directory -# that ends with `/lib`, we add the corresponding `/bin` directory. -set(_runtime_dependencies_dirs ${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}) -if(WIN32) - set(_dep_dirs ${_runtime_dependencies_dirs}) - foreach(_dep_dir IN LISTS _dep_dirs) - if(_dep_dir MATCHES "/lib$") - get_filename_component(_full_dep_bin_dir "${_dep_dir}/../bin" ABSOLUTE) - list(APPEND _runtime_dependencies_dirs "${_full_dep_bin_dir}") - endif() - endforeach() -endif() - -# Install the runtime dependencies of the module, eventually by copying the -# libraries from the toolchain. -message(STATUS "Looking for dependencies of qi_python and qi in: ${_runtime_dependencies_dirs}") -file(GET_RUNTIME_DEPENDENCIES - LIBRARIES "${QIPYTHON_QI_TARGET_FILE}" - MODULES "${QIPYTHON_QI_PYTHON_TARGET_FILE}" - RESOLVED_DEPENDENCIES_VAR _runtime_dependencies - UNRESOLVED_DEPENDENCIES_VAR _unresolved_runtime_dependencies - CONFLICTING_DEPENDENCIES_PREFIX _conflicting_runtime_dependencies - # eay32 is a library that comes with OpenSSL. - PRE_INCLUDE_REGEXES ssl eay crypto icu boost - PRE_EXCLUDE_REGEXES .* - DIRECTORIES ${_runtime_dependencies_dirs}) - -# Processing potential conflicts in dependencies: if there are multiple -# available choices for one of the runtime dependencies, we prefer one -# that is in the toolchain. Otherwise we just fallback to the first -# available path for that dependency. -if(_conflicting_runtime_dependencies_FILENAMES) - message(STATUS "Some conflicts were found for dependencies of qi_python and qi: ${_conflicting_runtime_dependencies_FILENAMES}") - foreach(_filename IN LISTS _conflicting_runtime_dependencies_FILENAMES) - message(STATUS "Conflicts of dependency '${_filename}': ${_conflicting_runtime_dependencies_${_filename}}") - set(_dep) - foreach(_path IN LISTS _conflicting_runtime_dependencies_${_filename}) - if(QIPYTHON_TOOLCHAIN_DIR AND _path MATCHES "^${QIPYTHON_TOOLCHAIN_DIR}") - set(_dep "${_path}") - message(STATUS "Using file '${_dep}' for dependency '${_filename}' as it is in the toolchain.") - break() - endif() - endforeach() - if(NOT _dep) - list(GET _conflicting_runtime_dependencies_${_filename} 0 _dep) - message(STATUS "Fallback to the first available path '${_dep}' for dependency '${_filename}'.") - endif() - list(APPEND _runtime_dependencies "${_dep}") - endforeach() -endif() - -message(STATUS "The dependencies of qi_python and qi are: ${_runtime_dependencies}") -if(_unresolved_runtime_dependencies) - message(STATUS "Some dependencies of qi_python and qi are unresolved: ${_unresolved_runtime_dependencies}") -endif() - -if(UNIX OR APPLE) - find_program(_patchelf "patchelf") - if(_patchelf) - if(APPLE) - set(_patchelf_runtime_path "@loader_path") - elseif(UNIX) - set(_patchelf_runtime_path "$ORIGIN") - endif() - else() - message(WARNING "The target system seems to be using ELF binaries, but the `patchelf` \ -tool could not be found. The installed runtime dependencies might not have their runtime path \ -set correctly to run. You might want to install `patchelf` on your system.") - endif() -endif() - -foreach(_needed_dep IN LISTS _runtime_dependencies) - set(_dep "${_needed_dep}") - - # Some of the dependency libraries are symbolic links, and `file(INSTALL)` will - # copy the symbolic link instead of their target (the real library) by default. - # There is an option FOLLOW_SYMLINK_CHAIN that will copy both the library and - # the symlink, but then when we create an archive (a wheel for instance), - # symlinks are replaced by the file they point to. This results in libraries - # being copied multiple times in the same archive with different names. - # - # Therefore we have to detect symlink manually and copy the files ourselves. - while(IS_SYMLINK "${_dep}") - set(_symlink "${_dep}") - file(READ_SYMLINK "${_symlink}" _dep) - if(NOT IS_ABSOLUTE "${_dep}") - get_filename_component(_dir "${_symlink}" DIRECTORY) - set(_dep "${_dir}/${_dep}") - endif() - message(STATUS "Dependency ${_symlink} is a symbolic link, resolved to ${_dep}") - endwhile() - - file(INSTALL "${_dep}" DESTINATION "${_module_install_dir}") - get_filename_component(_dep_filename "${_dep}" NAME) - set(_installed_dep "${_module_install_dir}/${_dep_filename}") - - if(_patchelf) - message(STATUS "Set runtime path of runtime dependency \"${_installed_dep}\" to \"${_patchelf_runtime_path}\"") - file(TO_NATIVE_PATH "${_installed_dep}" _native_installed_dep) - execute_process(COMMAND "${_patchelf}" - # Sets the RPATH/RUNPATH or may replace the RPATH by a RUNPATH, depending on the platform. - --set-rpath "${_patchelf_runtime_path}" "${_native_installed_dep}") - endif() - - if(NOT "${_dep}" STREQUAL "${_needed_dep}") - get_filename_component(_needed_dep_filename "${_needed_dep}" NAME) - set(_installed_needed_dep "${_module_install_dir}/${_needed_dep_filename}") - message(STATUS "Renaming ${_installed_dep} into ${_installed_needed_dep}") - file(RENAME "${_installed_dep}" "${_installed_needed_dep}") - list(TRANSFORM CMAKE_INSTALL_MANIFEST_FILES - REPLACE "${_installed_dep}" "${_installed_needed_dep}") - endif() -endforeach() diff --git a/cmake/native.py.in b/cmake/native.py.in deleted file mode 100644 index 351d698..0000000 --- a/cmake/native.py.in +++ /dev/null @@ -1 +0,0 @@ -__version__ = "@NATIVE_VERSION@" \ No newline at end of file diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake deleted file mode 100644 index 46dc882..0000000 --- a/cmake/set_dependencies.cmake +++ /dev/null @@ -1,306 +0,0 @@ -# This file finds the dependencies required by the project and creates the -# needed targets. -# -# It can only be included after a call to `project` and only once. -# -# After using CMake standard `find_package` modules, we often set the following -# variables (with the name of a dependency): -# - _PACKAGE_FOUND -# - _LIBRARIES -# - _INCLUDE_DIRS -# -# These variables are the ones the qibuild CMake part uses in its internal -# functions and `find_package` modules (such as `qi_use_lib`). This allows us -# to use CMake standard modules and bypass qibuild own search procedures. -# -# Once this file is loaded, the following targets are expected to be defined: -# -# - Threads::Threads, the thread library. -# - OpenSSL::SSL, aka libssl. -# - OpenSSL::Crypto, aka libcrypto. -# - Python::Interpreter, the Python interpreter, if found. -# - Python::Python, the Python library for embedding the interpreter. -# - Python::Module, the Python library for modules. -# - python_headers, header-only library for Python. -# - Boost::headers, the target for all the header-only libraries. -# - Boost::atomic, aka libboost_atomic. -# - Boost::date_time, aka libboost_date_time. -# - Boost::thread, aka libboost_thread. -# - Boost::chrono, aka libboost_chrono. -# - Boost::filesystem, aka libboost_filesystem. -# - Boost::locale, aka libboost_locale. -# - Boost::regex, aka libboost_regex. -# - Boost::program_options, aka libboost_program_options. -# - Boost::random, aka libboost_random. -# - pybind11, a header-only Python binding library. -# - gtest & gtest_main, the googletest library. -# - gmock & gmock_main, the googlemock library. -# - qi.interface, the high level target for linking with libqi. -# -# When needed, the following targets can also be defined: -# - ICU::i18n, aka libicui18n. -# - ICU::data, aka libicudata. -# - ICU::uc, aka libicuuc. - -include_guard(GLOBAL) - -# Version of Boost to use. It will be used as the argument to the `find_package(Boost)` call. -overridable_variable(BOOST_VERSION 1.77) - -# Version of pybind11 to use. -overridable_variable(PYBIND11_VERSION 2.9.0) - -# URL of the git repository from which to download pybind11. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. -overridable_variable(PYBIND11_GIT_REPOSITORY https://github.com/pybind/pybind11) - -# Git branch name, tag or commit hash to checkout for pybind11. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_TAG` argument. -overridable_variable(PYBIND11_GIT_TAG v${PYBIND11_VERSION}) - -# URL of the git repository from which to download googletest. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. -overridable_variable(GOOGLETEST_GIT_REPOSITORY https://github.com/google/googletest.git) - -# Git branch name, tag or commit hash to checkout for googletest. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_TAG` argument. -overridable_variable(GOOGLETEST_GIT_TAG main) - -set(PYTHON_VERSION_STRING "" CACHE STRING "Version of Python to look for. This \ -variable can be specified by tools run directly from Python to enforce the \ -use of the same version.") -mark_as_advanced(PYTHON_VERSION_STRING) - -if(NOT COMMAND FetchContent_Declare) - include(FetchContent) -endif() - -# This part is a bit tricky. Some of our internal toolchains tend to confuse the -# CMake standard `find_package` modules, so we try to use some heuristics to -# set hints for these modules. -# -# If we're using a toolchain, it's most likely either one of our internal -# desktop or Yocto toolchain. This should cover most of our building cases, but -# not all of them, especially for users trying to build the library outside of -# the organization. For these, we let them manually specify the needed -# dependencies path. -if(CMAKE_TOOLCHAIN_FILE) - message(VERBOSE "Toolchain file is set.") - get_filename_component(QIPYTHON_TOOLCHAIN_DIR "${CMAKE_TOOLCHAIN_FILE}" DIRECTORY) - get_filename_component(QIPYTHON_TOOLCHAIN_DIR "${QIPYTHON_TOOLCHAIN_DIR}" ABSOLUTE) - message(VERBOSE "Toolchain directory is ${QIPYTHON_TOOLCHAIN_DIR}.") - - # Cross-compiling for Yocto - if(YOCTO_SDK_TARGET_SYSROOT) - message(VERBOSE "Yocto cross compilation detected.") - message(VERBOSE "Yocto target sysroot is '${YOCTO_SDK_TARGET_SYSROOT}'.") - set(_yocto_target_sysroot_usr "${YOCTO_SDK_TARGET_SYSROOT}/usr") - set(_openssl_root ${_yocto_target_sysroot_usr}) - set(_icu_root ${_yocto_target_sysroot_usr}) - set(_boost_root ${_yocto_target_sysroot_usr}) - set(_python_root ${_yocto_target_sysroot_usr}) - - # Probably compiling for desktop - else() - message(VERBOSE "Assuming desktop compilation.") - set(_openssl_root "${QIPYTHON_TOOLCHAIN_DIR}/openssl") - set(_icu_root "${QIPYTHON_TOOLCHAIN_DIR}/icu") - set(_boost_root "${QIPYTHON_TOOLCHAIN_DIR}/boost") - # No definition of _python_root for desktop, we try to use the one - # installed on the system. - endif() - - # These variables are recognized by the associated CMake standard - # `find_package` modules (see their specific documentation for details). - # - # Setting them as cache variable enables user customisation. - set(OPENSSL_ROOT_DIR ${_openssl_root} CACHE PATH "root directory of an OpenSSL installation") - set(ICU_ROOT ${_icu_root} CACHE PATH "the root of the ICU installation") - set(BOOST_ROOT ${_boost_root} CACHE PATH "Boost preferred installation prefix") - - if(_python_root AND EXISTS "${_python_root}") - # The `qibuild` `python3-config.cmake` module must be used to find the - # dependency as it is packaged in our internal toolchains. - # The CMake standard `find_package` module is named `Python3`, looking for - # `PYTHON3` forces CMake to use the module that comes with `qibuild`. - find_package(PYTHON3 - REQUIRED CONFIG - NAMES PYTHON3 Python3 python3 - HINTS ${_python_root}) - - if(PYTHON3_FOUND OR PYTHON3_PACKAGE_FOUND) - # We want to set the `Python_LIBRARY` variable that the FindPython CMake module - # recognizes, in order to instruct it where to find the library (the FindPython - # module search mechanism uses the python-config utility which is not packaged - # well enough in our toolchains for it to work, therefore we have to manually - # tell the module where the library is). - # - # The `qibuild` Python3 module fills the PYTHON3_LIBRARIES variable with the - # format "general;;debug;", which is recognized by - # `target_link_libraries` (see the function documentation for - # details). However, the FindPython CMake module expects an absolute path. - # - # Therefore we create an interface library `Python_interface`, we use - # `target_link_libraries` so that the specific format is parsed, and then - # we get the full path of the library from the `LINK_LIBRARIES` property of - # the target. - add_library(Python_interface INTERFACE) - target_link_libraries(Python_interface INTERFACE ${PYTHON3_LIBRARIES}) - get_target_property(Python_LIBRARY Python_interface INTERFACE_LINK_LIBRARIES) - list(GET PYTHON3_INCLUDE_DIRS 0 Python_INCLUDE_DIR) - endif() - endif() -endif() - - -# C++17 -add_library(cxx17 INTERFACE) -target_compile_features(cxx17 INTERFACE cxx_std_17) - - -# Threads -find_package(Threads REQUIRED) -if(CMAKE_USE_PTHREADS_INIT) - set(PTHREAD_PACKAGE_FOUND TRUE) - set(PTHREAD_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) - set(PTHREAD_INCLUDE_DIRS) -endif() - - -# OpenSSL -find_package(OpenSSL REQUIRED) - - -# Python -parse_python_version(_python_version "${PYTHON_VERSION_STRING}") -if(_python_version_VERSION_STRING) - set(_python_version ${_python_version_VERSION_RELEASE_MAJOR_MINOR}) -else() - set(_python_version 3.5) # default to Python 3.5+. -endif() - -# Set variables that the CMake module recognizes from variables that the -# scikit-build build system module may pass to us. -if(PYTHON_LIBRARY) - set(Python_LIBRARY "${PYTHON_LIBRARY}") -endif() -if(PYTHON_INCLUDE_DIR) - set(Python_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") -endif() - -# First search for Python with Interpreter+Devel components, to allow the module -# to look for the interpreter that goes with the Python development library. -# Then if it could not find both, try looking for the development component -# only, but this time force the module to find it or fail (REQUIRED). -# Some of our toolchains (when crosscompiling for instance) have the Python -# library but not an interpreter that can run on the host. -find_package(Python ${_python_version} COMPONENTS Development Interpreter) -if(NOT Python_FOUND) - find_package(Python ${_python_version} REQUIRED COMPONENTS Development) -endif() - -list(GET Python_LIBRARIES 0 _py_lib) -add_library(python_headers INTERFACE) -target_include_directories(python_headers INTERFACE ${Python_INCLUDE_DIRS}) -if(_py_lib MATCHES "python([23])\\.([0-9]+)([mu]*d[mu]*)") - target_compile_definitions(python_headers INTERFACE Py_DEBUG) -endif() - - -# Boost -set(_boost_comps atomic date_time thread chrono filesystem locale regex - program_options random) -find_package(Boost ${BOOST_VERSION} EXACT REQUIRED COMPONENTS ${_boost_comps}) - -set(BOOST_PACKAGE_FOUND TRUE) -set(BOOST_LIBRARIES ${Boost_LIBRARIES}) -set(BOOST_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) -# Also set the "qibuild variables" for each of the components of boost. -foreach(_comp IN LISTS _boost_comps) - string(TOUPPER ${_comp} _uc_comp) - set(BOOST_${_uc_comp}_PACKAGE_FOUND TRUE) - set(BOOST_${_uc_comp}_LIBRARIES ${Boost_${_uc_comp}_LIBRARY}) - set(BOOST_${_uc_comp}_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) -endforeach() - - -# ICU. These are the components that might be needed by Boost::locale. -find_package(ICU COMPONENTS i18n uc data) -set(ICU_PACKAGE_FOUND ${ICU_FOUND}) -if(ICU_FOUND) - target_link_libraries(Boost::locale INTERFACE ICU::i18n ICU::data ICU::uc) -endif() - - -# pybind11, which is a header-only library. -# It's simpler to just use it as a CMake external project (through FetchContent) -# and download it directly into the build directory, than to try to use a -# preinstalled version through `find_package` -FetchContent_Declare(pybind11 - GIT_REPOSITORY ${PYBIND11_GIT_REPOSITORY} - GIT_TAG ${PYBIND11_GIT_TAG}) -FetchContent_GetProperties(pybind11) -if(NOT pybind11_POPULATED) - FetchContent_Populate(pybind11) - # pybind11 build system and targets do a lot of stuff automatically (such as - # looking for Python libs, adding the C++11 flag, ...). We prefer to do things - # ourselves, so we recreate a target directly instead of adding it as a - # subproject. - add_library(pybind11 INTERFACE) - target_include_directories(pybind11 INTERFACE ${pybind11_SOURCE_DIR}/include) - target_link_libraries(pybind11 INTERFACE cxx17 python_headers) -endif() - - -# googletest -# -# This is the simplest way to depend on googletest & googlemock. As they are -# fairly quick to compile, we can have it as a subproject (as a `FetchContent` -# project). -FetchContent_Declare(googletest - GIT_REPOSITORY ${GOOGLETEST_GIT_REPOSITORY} - GIT_TAG ${GOOGLETEST_GIT_TAG}) -FetchContent_GetProperties(googletest) -if(NOT googletest_POPULATED) - FetchContent_Populate(googletest) - set(INSTALL_GTEST OFF) - add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) -endif() - -set(GTEST_PACKAGE_FOUND TRUE) -set(GMOCK_PACKAGE_FOUND TRUE) -set(GTEST_LIBRARIES $) -set(GMOCK_LIBRARIES $) - -# Unfortunately, we cannot get the `INTERFACE_INCLUDE_DIRECTORIES` of the gtest -# or gmock targets as they might contain generator expressions containing ";" -# characters, which qibuild wrongly splits as elements of a list. Instead, we -# have to build the paths manually. -set(GTEST_INCLUDE_DIRS ${googletest_SOURCE_DIR}/googletest/include) -set(GMOCK_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} - ${googletest_SOURCE_DIR}/googlemock/include) - - -# LibQi -include(set_libqi_dependency) - -# Generate a list of dependency directories that can be used for -# instance to generate RPATH or install those dependencies. -set(QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS ${Boost_LIBRARY_DIRS}) -foreach(_lib IN LISTS OPENSSL_LIBRARIES ICU_LIBRARIES) - if(EXISTS ${_lib}) - get_filename_component(_dir ${_lib} DIRECTORY) - list(APPEND QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS ${_dir}) - endif() -endforeach() -list(REMOVE_DUPLICATES QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS) - -# Adds all the dependency directories as RPATH for a target ouput -# binary in the build directory, so that we may for instance execute -# it directly. -function(set_build_rpath_to_qipython_dependencies target) - message(VERBOSE - "Setting ${target} build RPATH to '${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}'") - set_property(TARGET ${target} PROPERTY - BUILD_RPATH ${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}) -endfunction() diff --git a/cmake/set_globals.cmake b/cmake/set_globals.cmake deleted file mode 100644 index 3bbccd9..0000000 --- a/cmake/set_globals.cmake +++ /dev/null @@ -1,50 +0,0 @@ -include_guard(GLOBAL) - -# Convert relative paths in `target_sources` to absolute. -cmake_policy(SET CMP0076 NEW) - -# Set the default policies for subprojects using an older CMake version -# (specified through `cmake_minimum_required`). Setting the policy directly -# (through `cmake_policy`) is not enough as it will be overwritten by the call -# to `cmake_minimum_required` in the subproject. This is to remove spurious -# warnings we get otherwise. -# -# 0048 - project() command manages VERSION variables. -set(CMAKE_POLICY_DEFAULT_CMP0048 NEW) -# 0056 - Honor link flags in try_compile() source-file signature. -set(CMAKE_POLICY_DEFAULT_CMP0056 NEW) -# 0066 - Honor per-config flags in try_compile() source-file signature. -set(CMAKE_POLICY_DEFAULT_CMP0066 NEW) -# 0060 - Link libraries by full path even in implicit directories. -set(CMAKE_POLICY_DEFAULT_CMP0060 NEW) -# 0063 - Honor visibility properties for all target types. -set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) -# 0074 - find_package uses PackageName_ROOT variables. -set(CMAKE_POLICY_DEFAULT_CMP0074 NEW) -# 0077 - option() honors normal variables. -set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) -# 0082 - Install rules from add_subdirectory() are interleaved with those in caller. -set(CMAKE_POLICY_DEFAULT_CMP0082 NEW) - -# Do not export symbols by default on new targets. -set(CMAKE_CXX_VISIBILITY_PRESET hidden) - -# We use RPATH on systems that handle them. -set(CMAKE_SKIP_RPATH OFF) -set(CMAKE_SKIP_BUILD_RPATH OFF) - -# Do not use the installation RPATH/RUNPATH values for the build directory. -# Our build directories do not necessarily share the same structure as our install -# directories. -set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF) - -# Do not append to the runtime search path (rpath) of installed binaries any -# directory outside the project that is in the linker search path or contains -# linked library files. -# Instead we copy the dependencies we need in the install directory, and we set -# the right RPATH ourselves. -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH OFF) - -set(QIPYTHON_PYTHON_MODULE_NAME qi) -set(QIPYTHON_PYTHON_MODULE_SOURCE_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/${QIPYTHON_PYTHON_MODULE_NAME}) \ No newline at end of file diff --git a/cmake/set_install_rules.cmake b/cmake/set_install_rules.cmake deleted file mode 100644 index 583a400..0000000 --- a/cmake/set_install_rules.cmake +++ /dev/null @@ -1,7 +0,0 @@ -include_guard(GLOBAL) - -if(QIPYTHON_STANDALONE) - include(set_install_rules_standalone) -else() - include(set_install_rules_system) -endif() diff --git a/cmake/set_install_rules_standalone.cmake b/cmake/set_install_rules_standalone.cmake deleted file mode 100644 index e70f3d6..0000000 --- a/cmake/set_install_rules_standalone.cmake +++ /dev/null @@ -1,57 +0,0 @@ -include_guard(GLOBAL) - -if(NOT QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS) - message(AUTHOR_WARNING "The QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS is empty.") -endif() - -# Stripping the binaries enables us to have lighter wheels. -# -# CMake offers a target "install/strip" that normally does that automatically, -# but scikit-build uses the plain "install" target and this behavior is not -# easily customisable. So this option exists so that we may manually strip the -# binaries when set. -option(QIPYTHON_FORCE_STRIP - "If set, forces the stripping of the qi_python module at install." - TRUE) - -if(${QIPYTHON_FORCE_STRIP}) - message(STATUS "The qi_python library will be stripped at install.") - install(CODE "set(CMAKE_INSTALL_DO_STRIP TRUE)" - COMPONENT Module) -endif() - -# qibuild automatically installs a `path.conf` file that we don't really care -# about in our wheel, and we have no other way to disable its installation other -# than letting it be installed then removing it. -install(CODE - "set(_path_conf \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/share/qi/path.conf\") - file(REMOVE \${_path_conf}) - list(REMOVE_ITEM CMAKE_INSTALL_MANIFEST_FILES \${_path_conf})" - COMPONENT runtime) - -if(QIPYTHON_TOOLCHAIN_DIR) - install(CODE "set(QIPYTHON_TOOLCHAIN_DIR \"${QIPYTHON_TOOLCHAIN_DIR}\")" - COMPONENT Module) -endif() - -install(CODE - "set(QIPYTHON_PYTHON_MODULE_NAME \"${QIPYTHON_PYTHON_MODULE_NAME}\") - set(QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS \"${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}\") - set(QIPYTHON_QI_PYTHON_TARGET_FILE \"$\") - set(QIPYTHON_QI_TARGET_FILE \"$\") - set(QIPYTHON_QI_PYTHON_TARGET_FILE_NAME \"$\") - set(QIPYTHON_QI_TARGET_FILE_NAME \"$\")" - COMPONENT Module) -install(SCRIPT cmake/install_runtime_dependencies.cmake COMPONENT Module) - -set_install_rpath(TARGETS qi_python qi ORIGIN) -install(TARGETS qi_python qi - LIBRARY DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - # On Windows, shared libraries (DLL) are considered `RUNTIME`. - RUNTIME DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - COMPONENT Module) - -# Install the Python file containing native informations. -install(FILES "${QIPYTHON_NATIVE_PYTHON_FILE}" - DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - COMPONENT Module) \ No newline at end of file diff --git a/cmake/set_install_rules_system.cmake b/cmake/set_install_rules_system.cmake deleted file mode 100644 index b7d8e73..0000000 --- a/cmake/set_install_rules_system.cmake +++ /dev/null @@ -1,35 +0,0 @@ -include_guard(GLOBAL) - - -if(NOT Python_VERSION_MAJOR OR NOT Python_VERSION_MINOR) - message(AUTHOR_WARNING "The Python major or minor version is unknown.") -endif() -set(_sitelib_dir - "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages/${QIPYTHON_PYTHON_MODULE_NAME}") - -# Set the RPATH of the module to a relative libraries directory. We assume that, -# if the module is installed in "/path/lib/pythonX.Y/site-packages/qi", the -# libraries we need are instead in "/path/lib", so we set the RPATH accordingly. -set_install_rpath(TARGETS qi_python ORIGIN PATHS_REL_TO_ORIGIN "../../..") - -install(TARGETS qi_python - LIBRARY DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -install(FILES ${QIPYTHON_PYTHON_MODULE_FILES} - DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -# Install the Python file containing native information. -install(FILES "${QIPYTHON_NATIVE_PYTHON_FILE}" - DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -# Set the RPATH of the plugin to a relative libraries directory. We assume that, -# if the plugin is installed in "/path/lib/qi/plugins", the -# libraries we need are instead in "/path/lib", so we set the RPATH accordingly. -set_install_rpath(TARGETS qimodule_python_plugin ORIGIN PATHS_REL_TO_ORIGIN "../..") - -install(TARGETS qimodule_python_plugin - LIBRARY DESTINATION "lib/qi/plugins" - COMPONENT runtime) \ No newline at end of file diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake deleted file mode 100644 index ed2fa99..0000000 --- a/cmake/set_libqi_dependency.cmake +++ /dev/null @@ -1,53 +0,0 @@ -overridable_variable(LIBQI_VERSION 4.0.0) - -# Our github clone is sometimes late or is missing tags, so we enable -# customisation of the URL at configuration time, so users can use another clone. -overridable_variable(LIBQI_GIT_REPOSITORY https://github.com/aldebaran/libqi.git) - -overridable_variable(LIBQI_GIT_TAG qi-framework-v${LIBQI_VERSION}) - -if(LIBQI_VERSION) - message(STATUS "LibQi: expected version is \"${LIBQI_VERSION}\"") -endif() - -# Checks that the version present in the `package_xml_file` file is the same as -# the one specified in the `LIBQI_VERSION` cache variable, if it is set. -function(check_libqi_version package_xml_file version_var) - file(STRINGS ${package_xml_file} _package_xml_strings REGEX version) - set(_package_version "") - if(_package_xml_strings MATCHES [[version="([^"]*)"]]) - set(_package_version ${CMAKE_MATCH_1}) - message(STATUS "LibQi: found version \"${_package_version}\" from file '${package_xml_file}'") - if(LIBQI_VERSION AND NOT (LIBQI_VERSION VERSION_EQUAL _package_version)) - message(FATAL_ERROR "LibQi version mismatch (expected \ - \"${LIBQI_VERSION}\", found \"${_package_version}\")") - endif() - else() - set(_msg_type WARNING) - if (LIBQI_VERSION) - set(_msg_type FATAL_ERROR) - endif() - message(${_msg_type} "Could not read LibQi version from file \ -${package_xml_file}. Please check that the file contains a version \ -attribute.") - endif() - set(${version_var} ${_package_version} PARENT_SCOPE) -endfunction() - -if(QIPYTHON_STANDALONE) - include(set_libqi_dependency_standalone) -else() - include(set_libqi_dependency_system) -endif() - -target_link_libraries(qi.interface INTERFACE cxx17) - -# Generate a Python file containing information about the native part of the -# module.\ -set(NATIVE_VERSION "${LIBQI_PACKAGE_VERSION}") -if(NOT NATIVE_VERSION) - set(NATIVE_VERSION "unknown") -endif() -set(QIPYTHON_NATIVE_PYTHON_FILE - "${CMAKE_CURRENT_BINARY_DIR}/${QIPYTHON_PYTHON_MODULE_NAME}/native.py") -configure_file(cmake/native.py.in "${QIPYTHON_NATIVE_PYTHON_FILE}" @ONLY) diff --git a/cmake/set_libqi_dependency_standalone.cmake b/cmake/set_libqi_dependency_standalone.cmake deleted file mode 100644 index a7de9f9..0000000 --- a/cmake/set_libqi_dependency_standalone.cmake +++ /dev/null @@ -1,71 +0,0 @@ -# In standalone mode, LibQi is currently added as a subproject. It's a -# controversial choice but depending on a prebuilt configuration, installation -# or package is messy with our CMake code. It makes compilation longer but it -# ensures consistency. - -include_guard(GLOBAL) - -if(NOT COMMAND FetchContent_Declare) - include(FetchContent) -endif() - -if(FETCHCONTENT_SOURCE_DIR_LIBQI) - message(STATUS "LibQi: using sources at \"${FETCHCONTENT_SOURCE_DIR_LIBQI}\"") -else() - message(STATUS "LibQi: using sources from \"${LIBQI_GIT_REPOSITORY}\" at tag \ -\"${LIBQI_GIT_TAG}\"") -endif() - -FetchContent_Declare(LibQi - GIT_REPOSITORY "${LIBQI_GIT_REPOSITORY}" - GIT_TAG "${LIBQI_GIT_TAG}" -) -FetchContent_GetProperties(LibQi) -if(NOT libqi_POPULATED) - FetchContent_Populate(LibQi) -endif() - -check_libqi_version("${libqi_SOURCE_DIR}/package.xml" LIBQI_PACKAGE_VERSION) - -# Adds LibQi as a subproject which will create a `qi` target. -# -# We use a function in order to scope the value of a few variables only for the -# LibQi subproject. -function(add_libqi_subproject) - # Do not build examples and tests, we don't need them in this project. - set(BUILD_EXAMPLES OFF) - set(QI_WITH_TESTS OFF) - - # Add it as a subproject but use `EXCLUDE_FROM_ALL` to disable the installation - # rules of the subproject files.com - add_subdirectory(${libqi_SOURCE_DIR} ${libqi_BINARY_DIR} EXCLUDE_FROM_ALL) -endfunction() -add_libqi_subproject() - -add_library(qi.interface INTERFACE) - -# Add the subproject include directories as "system" to avoid warnings -# generated by LibQi headers. -target_include_directories(qi.interface SYSTEM INTERFACE - ${libqi_SOURCE_DIR} - ${libqi_BINARY_DIR}/include) - -target_link_libraries(qi.interface INTERFACE - qi - -# qibuild CMake macros fail to propagate LibQi's include directories to the -# interface of the `qi` target, so we have to add them ourselves. For Boost, -# adding the header-only target is sufficient, as it will add the include -# directories, and the link libraries are already part of the `qi` target's -# interface (qibuild propagates those fine). - Boost::boost - -# These targets are not strictly speaking libraries but they add flags that -# disable autolinking in Boost and force dynamic linking, which are required -# on Windows. - Boost::disable_autolinking - Boost::dynamic_linking - -# For the libraries other than Boost, we don't have a header-only target to -# only add the include directories, so we have to directly link to them. - OpenSSL::SSL OpenSSL::Crypto) \ No newline at end of file diff --git a/cmake/set_libqi_dependency_system.cmake b/cmake/set_libqi_dependency_system.cmake deleted file mode 100644 index 46d2168..0000000 --- a/cmake/set_libqi_dependency_system.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# In system mode, LibQi must be accessible through find_package. -find_package(qi REQUIRED CONFIG) - -# If the qi package was found from a build directory in LibQi sources (which -# is the case when built with qibuild), then we expect that `qi_DIR` is set -# to the path `<...>/libqi/build/sdk/cmake`. -# If it was found in one of our toolchains, then we expect that `qi_DIR` is -# set to the path `/libqi(or qi-framework)/share/cmake/qi` in which -# case the `package.xml` file is in `/libqi(or qi-framework)`. -get_filename_component(QIPYTHON_LIBQI_PACKAGE_FILE - "${qi_DIR}/../../../package.xml" ABSOLUTE) -if(NOT EXISTS "${QIPYTHON_LIBQI_PACKAGE_FILE}") - get_filename_component(QIPYTHON_LIBQI_PACKAGE_FILE - "${qi_DIR}/../../package.xml" ABSOLUTE) -endif() - -set(QIPYTHON_LIBQI_PACKAGE_FILE "${QIPYTHON_LIBQI_PACKAGE_FILE}" CACHE FILEPATH - "Path to the `package.xml` file of LibQi.") - -if(LIBQI_VERSION) - if(EXISTS "${QIPYTHON_LIBQI_PACKAGE_FILE}") - check_libqi_version("${QIPYTHON_LIBQI_PACKAGE_FILE}" LIBQI_PACKAGE_VERSION) - else() - message(WARNING "The `package.xml` file for LibQi could not be found at \ -'${QIPYTHON_LIBQI_PACKAGE_FILE}' (the LibQi package was found in '${qi_DIR}'). \ -The version of LibQi could not be verified. Please manually specify a valid \ -path as the 'QIPYTHON_LIBQI_PACKAGE_FILE' variable to fix this issue, or unset \ -the 'LIBQI_VERSION' variable to completely disable this check.") - endif() -endif() - -add_library(qi.interface INTERFACE) -target_include_directories(qi.interface SYSTEM INTERFACE ${QI_INCLUDE_DIRS}) -target_link_libraries(qi.interface INTERFACE ${QI_LIBRARIES}) - -foreach(_dep IN LISTS QI_DEPENDS) - if(NOT ${_dep}_PACKAGE_FOUND) - find_package(${_dep} REQUIRED CONFIG) - endif() - target_include_directories(qi.interface SYSTEM INTERFACE ${${_dep}_INCLUDE_DIRS}) - target_link_libraries(qi.interface INTERFACE ${${_dep}_LIBRARIES}) -endforeach() diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..6777ae0 --- /dev/null +++ b/conanfile.py @@ -0,0 +1,97 @@ +from conan import ConanFile, tools +from conan.tools.cmake import cmake_layout + +BOOST_COMPONENTS = [ + "atomic", + "chrono", + "container", + "context", + "contract", + "coroutine", + "date_time", + "exception", + "fiber", + "filesystem", + "graph", + "graph_parallel", + "iostreams", + "json", + "locale", + "log", + "math", + "mpi", + "nowide", + "program_options", + "python", + "random", + "regex", + "serialization", + "stacktrace", + "system", + "test", + "thread", + "timer", + "type_erasure", + "wave", +] + +USED_BOOST_COMPONENTS = [ + "atomic", # required by thread + "chrono", # required by thread + "container", # required by thread + "date_time", # required by thread + "exception", # required by thread + "filesystem", # required by libqi + "locale", # required by libqi + "program_options", # required by libqi + "random", # required by libqi + "regex", # required by libqi + "system", # required by thread + "thread", +] + + +class QiPythonConan(ConanFile): + requires = [ + "boost/[~1.78]", + "pybind11/[~2.9]", + "qi/4.0.1", + ] + + test_requires = [ + "gtest/cci.20210126", + ] + + generators = "CMakeToolchain", "CMakeDeps" + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + + default_options = { + "boost/*:shared": True, + "openssl/*:shared": True, + "qi/*:with_boost_locale": True, # for `pytranslator.cpp` + } + + # Disable every components of Boost unless we actively use them. + default_options.update( + { + f"boost/*:without_{_name}": False + if _name in USED_BOOST_COMPONENTS + else True + for _name in BOOST_COMPONENTS + } + ) + + def layout(self): + # Configure the format of the build folder name, based on the value of some variables. + self.folders.build_folder_vars = [ + "settings.os", + "settings.arch", + "settings.compiler", + "settings.build_type", + ] + + # The cmake_layout() sets the folders and cpp attributes to follow the + # structure of a typical CMake project. + cmake_layout(self) diff --git a/pyproject.toml b/pyproject.toml index cc4c529..69a9dea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,49 @@ -[project] -dependencies = [ - "packaging" -] - # PEP 518 [build-system] -# Minimum requirements for the build system to execute. -requires = [ - "setuptools >= 47, < 51", # setuptools 51 dropped support for Python 3.5. - "wheel >= 0.34", # tested with 0.34. - "scikit-build >= 0.10", # tested with 0.10. - "cmake ~= 3.16", # CMake >= 3.16, CMake < 4. - "ninja ~= 1", # ninja >= 1, ninja < 2. - "toml ~= 10", - "packaging ~= 21", +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +# PEP 621 +[project] +name = "qi" +description = "LibQi Python bindings" +version = "3.1.3.dev0" +readme = "README.rst" +requires-python = ">=3.7" +license = { "file" = "COPYING" } +keywords=[ + "libqi", + "qi", + "naoqi", + "aldebaran", + "robotics", + "robot", + "nao", + "pepper", + "romeo", + "plato", +] +classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Embedded Systems", + "Framework :: Robot Framework :: Library", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", ] +maintainers = [ + { email = "framework@aldebaran.com" }, + { name = "Vincent Palancher", email = "vincent.palancher@aldebaran.com" }, + { name = "Jérémy Monnon", email = "jmonnon@aldebaran.com" }, +] + +[project.urls] +repository = "https://github.com/aldebaran/libqi-python" diff --git a/qi/__init__.py b/qi/__init__.py index 7f5f83e..1770583 100644 --- a/qi/__init__.py +++ b/qi/__init__.py @@ -5,14 +5,6 @@ """LibQi Python bindings.""" -import platform -from packaging import version - -py_version = version.parse(platform.python_version()) -min_version = version.parse('3.5') -if py_version < min_version: - raise RuntimeError('Python 3.5+ is required.') - import sys # noqa: E402 import atexit # noqa: E402 diff --git a/qi/_version.py b/qi/_version.py deleted file mode 100644 index f71b21a..0000000 --- a/qi/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '3.1.2' diff --git a/qi/_version.py.in b/qi/_version.py.in new file mode 100644 index 0000000..11ba4dd --- /dev/null +++ b/qi/_version.py.in @@ -0,0 +1 @@ +__version__ = "@QI_PYTHON_VERSION_STRING@" diff --git a/qi/native.py.in b/qi/native.py.in new file mode 100644 index 0000000..8a158e2 --- /dev/null +++ b/qi/native.py.in @@ -0,0 +1 @@ +__version__ = "@NATIVE_VERSION@" diff --git a/qiproject.xml b/qiproject.xml deleted file mode 100644 index b106649..0000000 --- a/qiproject.xml +++ /dev/null @@ -1,13 +0,0 @@ - - Joël Lamotte - Jérémy Monnon - Philippe Martin - Vincent Palancher - Julien Bernard - - - - - - - diff --git a/qipython.pml b/qipython.pml deleted file mode 100644 index b55798b..0000000 --- a/qipython.pml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/setup.py b/setup.py deleted file mode 100755 index 66c1e53..0000000 --- a/setup.py +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import platform -import pathlib -from setuptools import find_packages -from skbuild import setup -from packaging import version -import toml - -py_version = version.parse(platform.python_version()) -min_version = version.parse('3.5') -if py_version < min_version: - raise RuntimeError('Python 3.5+ is required.') - -# Parse `pyproject.toml` runtime dependencies. -pyproject_data = toml.loads(pathlib.Path('pyproject.toml').read_text()) -pyproject_deps = pyproject_data['project']['dependencies'] - -here = os.path.abspath(os.path.dirname(__file__)) - -version = {} -with open(os.path.join(here, "qi", "_version.py")) as f: - exec(f.read(), version) - -# Get the long description from the README file -with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='qi', - version=version['__version__'], - description='LibQi Python bindings', - long_description=long_description, - long_description_content_type='text/x-rst', - keywords=['libqi', 'qi', 'naoqi', - 'softbank', 'robotics', 'aldebaran', - 'robot', 'nao', 'pepper', 'romeo'], - url='https://github.com/aldebaran/libqi-python', - author='SoftBank Robotics Europe', - author_email='release@softbankrobotics.com', - license='BSD 3-Clause License', - python_requires='~=3.5', - install_requires=pyproject_deps, - packages=find_packages(exclude=[ - '*.test', '*.test.*', 'test.*', 'test' - ]), - include_package_data=True, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Application Frameworks', - 'Topic :: Software Development :: Embedded Systems', - 'Framework :: Robot Framework :: Library', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3 :: Only', - ], - cmake_args=['-DQIPYTHON_STANDALONE:BOOL=ON'], -) From bab25a701a3ce45c70c8313f479b72ed718187eb Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 9 Jun 2023 20:25:34 +0200 Subject: [PATCH 23/40] Bumps version to v3.1.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69a9dea..6e048cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build" [project] name = "qi" description = "LibQi Python bindings" -version = "3.1.3.dev0" +version = "3.1.3" readme = "README.rst" requires-python = ">=3.7" license = { "file" = "COPYING" } From 6a492e2127a3b53e0348e78ca9f694da07525d85 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Fri, 9 Jun 2023 20:29:29 +0200 Subject: [PATCH 24/40] Bumps version to v3.1.4.dev0. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e048cc..3c9d70d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build" [project] name = "qi" description = "LibQi Python bindings" -version = "3.1.3" +version = "3.1.4.dev0" readme = "README.rst" requires-python = ">=3.7" license = { "file" = "COPYING" } From 82b7aa5d3afc3ef55e03ab3e52f39163a6272177 Mon Sep 17 00:00:00 2001 From: Vincent PALANCHER Date: Thu, 27 Jul 2023 12:25:42 +0000 Subject: [PATCH 25/40] Add Gitlab CI to build wheels. This patch adds Gitlab CI to the project to build wheels for Linux for each supported version of Python. This build can be manually triggered for any commit, but is automatic when a "qi-python-vXXX" tag is pushed. Furthermore, when a tag is pushed on GitLab, wheels are published in the package registry and a release is created. --- .gitlab-ci.yml | 55 ++++++++++++++++++++ CMakeLists.txt | 94 +++++++++++++++++------------------ ci/cibuildwheel_before_all.sh | 37 ++++++++++++++ ci/conan/global.conf | 2 + ci/conan/profiles/default | 8 +++ pyproject.toml | 19 +++++++ 6 files changed, 168 insertions(+), 47 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100755 ci/cibuildwheel_before_all.sh create mode 100644 ci/conan/global.conf create mode 100644 ci/conan/profiles/default diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..61b2b8c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,55 @@ +stages: + - build + - publish + - release + +.if-tag: &if-tag + if: "$CI_COMMIT_TAG =~ /^qi-python-v/" + +build-wheel: + stage: build + rules: + # Allow manually building on a Git commit on any branch. + - if: $CI_COMMIT_BRANCH + when: manual + # Always build on a Git tag. + - <<: *if-tag + tags: + - docker + image: python:3.8 + variables: + LIBQI_REPOSITORY_URL: "https://gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST/qi/libqi" + script: + - curl -sSL https://get.docker.com/ | sh + - pip install cibuildwheel==2.14.1 + - cibuildwheel --output-dir wheelhouse + artifacts: + paths: + - wheelhouse/ + +publish-wheel: + stage: publish + image: python:latest + rules: + - <<: *if-tag + needs: + - build-wheel + script: + - pip install build twine + - python -m build + - TWINE_PASSWORD="${CI_JOB_TOKEN}" + TWINE_USERNAME=gitlab-ci-token + python -m twine upload + --repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" + wheelhouse/* + +create-release: + stage: release + rules: + - <<: *if-tag + script: + - echo "Releasing $CI_COMMIT_TAG." + release: + tag_name: $CI_COMMIT_TAG + name: 'lib$CI_COMMIT_TITLE' + description: '$CI_COMMIT_TAG_MESSAGE' diff --git a/CMakeLists.txt b/CMakeLists.txt index b2a5f22..064b39a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,16 +66,14 @@ endif() ############################################################################## # Find Python ############################################################################## -# First search for Python with Interpreter+Devel components, to allow the module -# to look for the interpreter that goes with the Python development library. -# Then if it could not find both, try looking for the development component -# only, but this time force the module to find it or fail (REQUIRED). -# Some of our toolchains (when crosscompiling for instance) have the Python -# library but not an interpreter that can run on the host. -find_package(Python COMPONENTS Development Interpreter) -if(NOT Python_FOUND) - find_package(Python REQUIRED COMPONENTS Development) -endif() +find_package( + Python REQUIRED + COMPONENTS + Development.Module + OPTIONAL_COMPONENTS + Interpreter + Development.Embed +) ############################################################################## # Find Boost @@ -323,47 +321,49 @@ if(BUILD_TESTING) qi::qi ) - add_executable(test_qipython) - target_sources( - test_qipython - PRIVATE - tests/common.hpp - tests/test_qipython.cpp - tests/test_guard.cpp - tests/test_types.cpp - tests/test_signal.cpp - tests/test_property.cpp - tests/test_object.cpp - tests/test_module.cpp - ) - target_link_libraries( - test_qipython + if(Python_Development.Embed_FOUND) + add_executable(test_qipython) + target_sources( + test_qipython + PRIVATE + tests/common.hpp + tests/test_qipython.cpp + tests/test_guard.cpp + tests/test_types.cpp + tests/test_signal.cpp + tests/test_property.cpp + tests/test_object.cpp + tests/test_module.cpp + ) + target_link_libraries( + test_qipython + PRIVATE + qi_python_objects + cxx_standard + Python::Python + Boost::headers + pybind11::pybind11 + GTest::gmock + ) + gtest_discover_tests(test_qipython) + + add_executable(test_qipython_local_interpreter) + target_sources( + test_qipython_local_interpreter + PRIVATE + tests/test_qipython_local_interpreter.cpp + ) + target_link_libraries( + test_qipython_local_interpreter PRIVATE - qi_python_objects - cxx_standard Python::Python - Boost::headers pybind11::pybind11 + qi_python_objects + cxx_standard GTest::gmock - ) - gtest_discover_tests(test_qipython) - - add_executable(test_qipython_local_interpreter) - target_sources( - test_qipython_local_interpreter - PRIVATE - tests/test_qipython_local_interpreter.cpp - ) - target_link_libraries( - test_qipython_local_interpreter - PRIVATE - Python::Python - pybind11::pybind11 - qi_python_objects - cxx_standard - GTest::gmock - ) - gtest_discover_tests(test_qipython_local_interpreter) + ) + gtest_discover_tests(test_qipython_local_interpreter) + endif() if(NOT Python_Interpreter_FOUND) message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") diff --git a/ci/cibuildwheel_before_all.sh b/ci/cibuildwheel_before_all.sh new file mode 100755 index 0000000..72ea5e3 --- /dev/null +++ b/ci/cibuildwheel_before_all.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -x -e + +PACKAGE=$1 + +pip install 'conan>=2' 'cmake>=3.23' ninja + +# Perl dependencies required to build OpenSSL. +yum install -y perl-IPC-Cmd perl-Digest-SHA + +# Install Conan configuration. +conan config install "$PACKAGE/ci/conan" + +# Clone and export libqi to Conan cache. +# Possible improvement: +# Avoid duplicating the version number here with the version +# defined in conanfile.py. +GIT_SSL_NO_VERIFY=true \ + git clone --depth=1 \ + --branch qi-framework-v4.0.1 \ + "$LIBQI_REPOSITORY_URL" \ + /work/libqi +conan export /work/libqi --version=4.0.1 + +# Install dependencies of libqi-python from Conan, including libqi. +# Only use the build_type as a variable for the build folder name, so +# that the generated CMake preset is named "conan-release". +# +# Build everything from sources, so that we do not reuse precompiled binaries. +# This is because the GLIBC from the manylinux images are often older than the +# ones that were used to build the precompiled binaries, which means the binaries +# cannot by executed. +conan install "$PACKAGE" \ + --build="*" \ + -c tools.build:skip_test=true \ + -c tools.cmake.cmaketoolchain:generator=Ninja \ + -c tools.cmake.cmake_layout:build_folder_vars=[\"settings.build_type\"] diff --git a/ci/conan/global.conf b/ci/conan/global.conf new file mode 100644 index 0000000..bbcc54a --- /dev/null +++ b/ci/conan/global.conf @@ -0,0 +1,2 @@ +core:default_profile=default +core:default_build_profile=default diff --git a/ci/conan/profiles/default b/ci/conan/profiles/default new file mode 100644 index 0000000..4ca0f95 --- /dev/null +++ b/ci/conan/profiles/default @@ -0,0 +1,8 @@ +[settings] +arch=x86_64 +build_type=Release +compiler=gcc +compiler.cppstd=gnu17 +compiler.libcxx=libstdc++ +compiler.version=10 +os=Linux diff --git a/pyproject.toml b/pyproject.toml index 3c9d70d..2c98a36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,3 +47,22 @@ maintainers = [ [project.urls] repository = "https://github.com/aldebaran/libqi-python" + +[tool.scikit-build] +# Rely only on CMake install, not on Python packages detection. +wheel.packages = [] + +[tool.cibuildwheel] +build = "cp*manylinux*x86_64" +build-frontend = "build" + +environment-pass = ["LIBQI_REPOSITORY_URL"] + +[tool.cibuildwheel.linux] +# Build using the manylinux2014 image +manylinux-x86_64-image = "manylinux2014" + +before-all = ["ci/cibuildwheel_before_all.sh {package}"] + +[tool.cibuildwheel.linux.config-settings] +"cmake.args" = ["--preset=conan-release"] From 9dbe864482b99b15a6236b4f91189d9331bf270d Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Thu, 27 Jul 2023 14:27:49 +0200 Subject: [PATCH 26/40] Upgrade pybind11 to any 2.9+ version. --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 6777ae0..9d5e3a5 100644 --- a/conanfile.py +++ b/conanfile.py @@ -54,7 +54,7 @@ class QiPythonConan(ConanFile): requires = [ "boost/[~1.78]", - "pybind11/[~2.9]", + "pybind11/[^2.9]", "qi/4.0.1", ] From f2e6d0aa3886f21f65eaf41ca154104b42fc08cf Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Thu, 21 Sep 2023 16:57:08 +0200 Subject: [PATCH 27/40] Replaces some `pybind11::handle` uses by type deduction. This change is done to prevent slicing `pybind11::object` values (or its subclasses) into `pybind11::handle`, which may result into handles to garbage collected objects. --- src/pyobject.cpp | 4 ++-- src/pystrand.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pyobject.cpp b/src/pyobject.cpp index ddd1a4f..4038ed3 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -333,7 +333,7 @@ Object toObject(const ::py::object& obj) : ObjectThreadingModel_SingleThread); const auto attrKeys = ::py::reinterpret_steal<::py::list>(PyObject_Dir(obj.ptr())); - for (const ::py::handle& pyAttrKey : attrKeys) + for (const auto& pyAttrKey : attrKeys) { QI_ASSERT_TRUE(pyAttrKey); QI_ASSERT_FALSE(pyAttrKey.is_none()); @@ -342,7 +342,7 @@ Object toObject(const ::py::object& obj) const auto attrKey = pyAttrKey.cast(); auto memberName = attrKey; - const ::py::object attr = obj.attr(pyAttrKey); + const auto attr = obj.attr(pyAttrKey); if (attr.is_none()) { qiLogVerbose() << "The object attribute '" << attrKey diff --git a/src/pystrand.cpp b/src/pystrand.cpp index 7da98c5..0e1b2d8 100644 --- a/src/pystrand.cpp +++ b/src/pystrand.cpp @@ -35,15 +35,15 @@ bool isUnboundMethod(const ::py::function& func, const ::py::object& object) GILAcquire lock; const auto dir = ::py::reinterpret_steal<::py::list>(PyObject_Dir(object.ptr())); - for (const ::py::handle attrName : dir) + for (const auto& attrName : dir) { - const ::py::handle attr = object.attr(attrName); + const auto attr = object.attr(attrName); QI_ASSERT_TRUE(attr); if (!PyMethod_Check(attr.ptr())) continue; - const ::py::handle unboundMethod = PyMethod_GET_FUNCTION(attr.ptr()); + const auto unboundMethod = ::py::reinterpret_borrow<::py::object>(PyMethod_GET_FUNCTION(attr.ptr())); if (func.is(unboundMethod)) return true; } From 95df33474131cf6c21d6473110c4d6bfd232dbc1 Mon Sep 17 00:00:00 2001 From: Vincent PALANCHER Date: Tue, 7 Nov 2023 15:08:25 +0000 Subject: [PATCH 28/40] Reworks GIL related types to prevent some forms of program termination. This change is a rework of some GIL related types in the library to prevent and avoid some unexpected crashes and program terminations, which would occur most often at process exit: - A new exception type named `InterpreterFinalizingException` is introduced. It is thrown by some functions in the library when an operation fails because the interpreter is currently finalizing. - `GILAcquire` constructor now throws a `InterpreterFinalizingException` when the interpreter is finalizing and therefore the GIL could not be acquired. - `GILRelease` destructor now does not reacquire the GIL when the interpreter is finalizing. - The `Guarded` and `GILGuardedObject` types are removed, and replaced with a `SharedObject` type. This type now wraps a Python object value as a shared reference-counted value that does not require the GIL to copy or move. The last copy of a `SharedObject` value will acquire the GIL to release the object, if possible, or do nothing (and leak it) if the GIL cannot be acquired. - The `DeleteInOtherThread` deleter type is removed. Its implementation was incorrect and it has become unneeded. - Destructors of `Application` and `ApplicationSession` do not use the `DeleterInOtherThread` deleter type anymore. The hypothesis is that the `DeleteInOtherThread` deleter was previously needed because `Application` values were wrapped in shared pointers and could be destroyed in callbacks executed by native threads. This is not the case anymore and instead, they destroy the application in the same thread that the Python `Application` object is garbage collected, which most likely is the interpreter main thread (due to `Application` being a singleton in the Python code, and its destruction is scheduled in an `atexit` callback). - A new `invokeCatchPythonError` is introduced. It acquires the GIL, then invokes a function with the given arguments, but catches any `pybind11::error_already_set` exception thrown during this invocation, and throws a `std::runtime_error` instead. - The `currentThreadHoldsGil` function now returns an optional denoting if the interpreter is initialized and not finalizing. Another function `gilExistsAndCurrentThreadHoldsIt` is introduced that returns just a bool. - The `ObjectDecRef` internal type is removed, as it is unused. This change mitigates the problem observed in issue #SW-4658. --- CMakeLists.txt | 1 + qipython/common.hpp | 24 ++ qipython/common.hxx | 37 +++ qipython/pyguard.hpp | 290 ++++++++++++---------- src/pyapplication.cpp | 5 +- src/pyasync.cpp | 20 +- src/pyfuture.cpp | 17 +- src/pyobject.cpp | 12 +- src/pysignal.cpp | 6 +- src/pytypes.cpp | 10 - tests/test_guard.cpp | 118 ++++----- tests/test_qipython_local_interpreter.cpp | 69 ++++- 12 files changed, 366 insertions(+), 243 deletions(-) create mode 100644 qipython/common.hxx diff --git a/CMakeLists.txt b/CMakeLists.txt index 064b39a..ddba9e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ target_sources( PUBLIC qipython/common.hpp + qipython/common.hxx qipython/pyapplication.hpp qipython/pyasync.hpp qipython/pyclock.hpp diff --git a/qipython/common.hpp b/qipython/common.hpp index 55fbf78..3335f66 100644 --- a/qipython/common.hpp +++ b/qipython/common.hpp @@ -125,6 +125,28 @@ inline boost::optional interpreterIsFinalizing() #endif } +/// Acquires the GIL, then invokes a function with the given arguments, but +/// catches any `pybind11::error_already_set` exception thrown during this +/// invocation, and throws a `std::runtime_error` instead. This is useful to +/// avoid the undefined behavior caused by the `what` member function of +/// `pybind11::error_already_set` when the GIL is not acquired. +template +auto invokeCatchPythonError(F&& f, Args&&... args); + +/// Exception type thrown when an operation fails because the interpreter is +/// finalizing. +/// +/// Some operations are forbidden during interpreter finalization as they may +/// terminate the thread they are called from. This exception allows stopping +/// the flow of execution in this case before such functions are called. +struct InterpreterFinalizingException : std::exception +{ + inline const char* what() const noexcept override + { + return "the interpreter is finalizing"; + } +}; + } // namespace py } // namespace qi @@ -203,4 +225,6 @@ namespace detail PYBIND11_DECLARE_HOLDER_TYPE(T, boost::shared_ptr); +#include + #endif // QIPYTHON_COMMON_HPP diff --git a/qipython/common.hxx b/qipython/common.hxx new file mode 100644 index 0000000..2618b47 --- /dev/null +++ b/qipython/common.hxx @@ -0,0 +1,37 @@ +/* +** Copyright (C) 2023 Aldebaran Robotics +** See COPYING for the license +*/ + +#pragma once + +#ifndef QIPYTHON_COMMON_HXX +#define QIPYTHON_COMMON_HXX + +#include +#include +#include + +namespace qi +{ +namespace py +{ + +template +auto invokeCatchPythonError(F&& f, Args&&... args) +{ + GILAcquire acquire; + try + { + return std::invoke(std::forward(f), std::forward(args)...); + } + catch (const pybind11::error_already_set& err) + { + throw std::runtime_error(err.what()); + } +} + +} // namespace py +} // namespace qi + +#endif // QIPYTHON_COMMON_HXX diff --git a/qipython/pyguard.hpp b/qipython/pyguard.hpp index 33ea618..f20ee1b 100644 --- a/qipython/pyguard.hpp +++ b/qipython/pyguard.hpp @@ -10,6 +10,7 @@ #include #include +#include #include namespace qi @@ -17,10 +18,25 @@ namespace qi namespace py { -/// Returns whether or not the GIL is currently held by the current thread. -inline bool currentThreadHoldsGil() +/// Returns whether or not the GIL is currently held by the current thread. If +/// the interpreter is not yet initialized or has been finalized, returns an +/// empty optional, as there is no GIL available. +inline boost::optional currentThreadHoldsGil() { - return PyGILState_Check() == 1; + // PyGILState_Check() returns 1 (success) before the creation of the GIL and + // after the destruction of the GIL. + if (Py_IsInitialized() == 1) { + const auto gilAcquired = PyGILState_Check() == 1; + return boost::make_optional(gilAcquired); + } + return boost::none; +} + +/// Returns whether or not the GIL exists (i.e the interpreter is initialized +/// and not finalizing) and is currently held by the current thread. +inline bool gilExistsAndCurrentThreadHoldsIt() +{ + return currentThreadHoldsGil().value_or(false); } /// DefaultConstructible Guard @@ -35,98 +51,6 @@ ka::ResultOf invokeGuarded(F&& f, Args&&... args) namespace detail { -// Guards the copy, construction and destruction of an object, and only when -// in case of copy construction the source object evaluates to true, and in case -// of destruction, the object itself evaluates to true. -// -// Explanation: -// pybind11::object only requires the GIL to be locked on copy and if the -// source is not null. Default construction does not require the GIL either. -template -class Guarded -{ - using Storage = - typename std::aligned_storage::type; - Storage _object; - - Object& object() { return reinterpret_cast(_object); } - const Object& object() const { return reinterpret_cast(_object); } - -public: - template::value, int> = 0> - explicit Guarded(Arg0&& arg0, Args&&... args) - { - Guard g; - new(&_object) Object(std::forward(arg0), std::forward(args)...); - } - - Guarded() - { - // Default construction, no GIL needed. - new(&_object) Object(); - } - - Guarded(Guarded& o) - { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - new(&_object) Object(o.object()); - } - - Guarded(const Guarded& o) - { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - new(&_object) Object(o.object()); - } - - Guarded(Guarded&& o) - { - new(&_object) Object(std::move(o.object())); - } - - ~Guarded() - { - boost::optional optGuard; - if (object()) - optGuard.emplace(); - object().~Object(); - } - - Guarded& operator=(const Guarded& o) - { - if (this == &o) - return *this; - - { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - object() = o.object(); - } - return *this; - } - - Guarded& operator=(Guarded&& o) - { - if (this == &o) - return *this; - - object() = std::move(o.object()); - return *this; - } - - Object& operator*() { return object(); } - const Object& operator*() const { return object(); } - Object* operator->() { return &object(); } - const Object* operator->() const { return &object(); } - explicit operator Object&() { return object(); } - explicit operator const Object&() const { return object(); } -}; - /// G == pybind11::gil_scoped_acquire || G == pybind11::gil_scoped_release template void pybind11GuardDisarm(G& guard) @@ -141,37 +65,52 @@ void pybind11GuardDisarm(G& guard) } // namespace detail /// RAII utility type that guarantees that the GIL is locked for the scope of -/// the lifetime of the object. +/// the lifetime of the object. If the GIL cannot be acquired (for example, +/// because the interpreter is finalizing), throws an `InterpreterFinalizingException` +/// exception. /// /// Objects of this type (or objects composed of them) must not be kept alive /// after the hand is given back to the interpreter. /// /// This type is re-entrant. /// -/// postcondition: `GILAcquire acq;` establishes -/// `(interpreterIsFinalizing() && *interpreterIsFinalizing()) || currentThreadHoldsGil()` +/// postcondition: `GILAcquire acq;` establishes `gilExistsAndCurrentThreadHoldsIt()` struct GILAcquire { inline GILAcquire() { - const auto optIsFinalizing = interpreterIsFinalizing(); - const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; - // `gil_scoped_acquire` is re-entrant by itself, so we don't need to check - // whether or not the GIL is already held by the current thread. - if (!definitelyFinalizing) - _acq.emplace(); - QI_ASSERT(definitelyFinalizing || currentThreadHoldsGil()); + if (gilExistsAndCurrentThreadHoldsIt()) + return; + + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (isFinalizing) + throw InterpreterFinalizingException(); + + _state = ::PyGILState_Ensure(); + QI_ASSERT(gilExistsAndCurrentThreadHoldsIt()); + } + + inline ~GILAcquire() + { + // Even if releasing the GIL while the interpreter is finalizing is allowed, it does + // require the GIL to be currently held. But we have no guarantee that this is the case, + // because the GIL may have been released since we acquired it, and we could not + // reacquire it after that (maybe the interpreter is in fact finalizing). + // Therefore, only release the GIL if it is currently held by this thread. + if (_state && gilExistsAndCurrentThreadHoldsIt()) + ::PyGILState_Release(*_state); } + GILAcquire(const GILAcquire&) = delete; GILAcquire& operator=(const GILAcquire&) = delete; private: - boost::optional _acq; + boost::optional _state; }; -/// RAII utility type that guarantees that the GIL is unlocked for the scope of -/// the lifetime of the object. +/// RAII utility type that (as a best effort) tries to ensure that the GIL is +/// unlocked for the scope of the lifetime of the object. /// /// Objects of this type (or objects composed of them) must not be kept alive /// after the hand is given back to the interpreter. @@ -179,16 +118,44 @@ struct GILAcquire /// This type is re-entrant. /// /// postcondition: `GILRelease rel;` establishes -/// `(interpreterIsFinalizing() && *interpreterIsFinalizing()) || !currentThreadHoldsGil()` +/// `interpreterIsFinalizing().value_or(false) || +/// !gilExistsAndCurrentThreadHoldsIt()` struct GILRelease { inline GILRelease() { - const auto optIsFinalizing = interpreterIsFinalizing(); - const auto definitelyFinalizing = optIsFinalizing && *optIsFinalizing; - if (!definitelyFinalizing && currentThreadHoldsGil()) + // Even if releasing the GIL while the interpreter is finalizing is allowed, + // it does require the GIL to be currently held. However, reacquiring the + // GIL is forbidden if finalization started. + // + // It may happen that we try to release the GIL from a function ('F') called + // by the Python interpreter, while it is holding the GIL. In this case, if + // we try to release it, then fail to reacquire it and return the hand to + // the interpreter, the interpreter will most likely terminate the process, + // as it expects to still be holding the GIL. Failure to reacquire the GIL + // can only happen if the interpreter started finalization, either before we + // released it, or while it was released. + // + // Fortunately, in this case, because the interpreter is busy executing the + // function 'F', it is not possible for it to start finalization until 'F' + // returns. This means that the only possible reason of failure of + // reacquisition of the GIL is because the interpreter was finalizing + // *before* calling 'F', therefore before we released to GIL. Therefore, to + // prevent this failure, we check if the interpreter is finalizing before + // releasing the GIL, and if it is, we do nothing, and the GIL stays held. + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (!isFinalizing && gilExistsAndCurrentThreadHoldsIt()) _release.emplace(); - QI_ASSERT(definitelyFinalizing || !currentThreadHoldsGil()); + QI_ASSERT(isFinalizing || !gilExistsAndCurrentThreadHoldsIt()); + } + + inline ~GILRelease() + { + // Reacquiring the GIL is forbidden when the interpreter is finalizing, as + // it may terminate the current thread. + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (_release && isFinalizing) + detail::pybind11GuardDisarm(*_release); } GILRelease(const GILRelease&) = delete; @@ -198,11 +165,83 @@ struct GILRelease boost::optional _release; }; -/// Wraps a pybind11::object value and locks the GIL on copy and destruction. +/// Wraps a Python object as a shared reference-counted value that does not +/// require the GIL to copy, move or assign to. /// -/// This is useful for instance to put pybind11 objects in lambda functions so -/// that they can be copied around safely. -using GILGuardedObject = detail::Guarded; +/// On destruction, it releases the object with the GIL acquired. However, if +/// the GIL cannot be acquired, the object is leaked. This is most likely +/// mitigated by the fact that if the GIL is not available, it means the object +/// already has been or soon will be garbage collected by interpreter +/// finalization. +template +class SharedObject +{ + static_assert(std::is_base_of_v, + "template parameter T must be a subclass of pybind11::object"); + + struct State + { + std::mutex mutex; + T object; + }; + std::shared_ptr _state; + + struct StateDeleter + { + inline void operator()(State* state) const + { + auto handle = state->object.release(); + delete state; + + // Do not lock the GIL if there is nothing to release. + if (!handle) + return; + + try + { + GILAcquire acquire; + handle.dec_ref(); + } + catch (const qi::py::InterpreterFinalizingException&) + { + // Nothing, the interpreter is finalizing. + } + } + }; + +public: + SharedObject() = default; + + inline explicit SharedObject(T object) + : _state(new State { std::mutex(), std::move(object) }, StateDeleter()) + { + } + + /// Copies the inner Python object value by incrementing its reference count. + /// + /// @pre: If the inner value is not null, the GIL must be acquired. + T inner() const + { + QI_ASSERT_NOT_NULL(_state); + std::scoped_lock lock(_state->mutex); + return _state->object; + } + + /// Takes the inner Python object value and leaves a null value in its place. + /// + /// Any copy of the shared object will now have a null inner value. + /// + /// This operation does not require the GIL as the reference count of the + /// object is preserved. + T takeInner() + { + QI_ASSERT_NOT_NULL(_state); + std::scoped_lock lock(_state->mutex); + /// Hypothesis: Moving a `pybind11::object` steals the object and preserves + /// its reference count and therefore does not require the GIL. + return ka::exchange(_state->object, {}); + } +}; /// Deleter that deletes the pointer outside the GIL. /// @@ -218,23 +257,6 @@ struct DeleteOutsideGIL } }; -/// Deleter that delays the destruction of an object to another thread. -struct DeleteInOtherThread -{ - template - void operator()(T* ptr) const - { - GILRelease unlock; - // `std::async` returns an object, that unless moved from will block when - // destroyed until the task is complete, which is unwanted behavior here. - // Therefore we just take the result of the `std::async` call into a local - // variable and then ignore it. - auto fut = std::async(std::launch::async, [](std::unique_ptr) {}, - std::unique_ptr(ptr)); - QI_IGNORE_UNUSED(fut); - } -}; - } // namespace py } // namespace qi diff --git a/src/pyapplication.cpp b/src/pyapplication.cpp index 315d842..e9a0ed0 100644 --- a/src/pyapplication.cpp +++ b/src/pyapplication.cpp @@ -83,7 +83,7 @@ void exportApplication(::py::module& m) GILAcquire lock; - class_>( + class_( m, "Application") .def(init(withArgcArgv<>([](int& argc, char**& argv) { GILRelease unlock; @@ -93,8 +93,7 @@ void exportApplication(::py::module& m) .def_static("run", &Application::run, call_guard()) .def_static("stop", &Application::stop, call_guard()); - class_>( + class_( m, "ApplicationSession") .def(init(withArgcArgv( diff --git a/src/pyasync.cpp b/src/pyasync.cpp index 01b6310..f39ca2e 100644 --- a/src/pyasync.cpp +++ b/src/pyasync.cpp @@ -38,19 +38,15 @@ ::py::object async(::py::function pyCallback, const MicroSeconds delay(usDelay); - GILGuardedObject guardCb(pyCallback); - GILGuardedObject guardArgs(std::move(args)); - GILGuardedObject guardKwargs(std::move(kwargs)); - + SharedObject sharedCb(pyCallback); + SharedObject sharedArgs(std::move(args)); + SharedObject sharedKwArgs(std::move(kwargs)); auto invokeCallback = [=]() mutable { - GILAcquire lock; - auto res = ::py::object((*guardCb)(**guardArgs, ***guardKwargs)).cast(); - // Release references immediately while we hold the GIL, instead of waiting - // for the lambda destructor to relock the GIL. - *guardKwargs = {}; - *guardArgs = {}; - *guardCb = {}; - return res; + GILAcquire acquire; + return invokeCatchPythonError( + sharedCb.takeInner(), + *sharedArgs.takeInner(), + **sharedKwArgs.takeInner()).cast(); }; // If there is a strand attached to that callable, we use it but we cannot use diff --git a/src/pyfuture.cpp b/src/pyfuture.cpp index 9c473c9..dc4c6fb 100644 --- a/src/pyfuture.cpp +++ b/src/pyfuture.cpp @@ -53,8 +53,8 @@ template std::function(Args...)> toContinuation(const ::py::function& cb) { GILAcquire lock; - GILGuardedObject guardedCb(cb); - auto callGuardedCb = [=](Args... args) mutable { + SharedObject sharedCb(cb); + auto callSharedCb = [=](Args... args) mutable { GILAcquire lock; const auto handleExcept = ka::handle_exception_rethrow( exceptionLogVerbose( @@ -62,17 +62,18 @@ std::function(Args...)> toContinuation(const ::py::function& cb) "An exception occurred while executing a future continuation"), ka::type_t<::py::object>{}); const auto pyRes = - ka::invoke_catch(handleExcept, *guardedCb, std::forward(args)...); - // Release references immediately while we hold the GIL, instead of waiting - // for the lambda destructor to relock the GIL. - *guardedCb = {}; + ka::invoke_catch(handleExcept, [&] { + return invokeCatchPythonError( + sharedCb.takeInner(), + std::forward(args)...); + }); return castIfNotVoid(pyRes); }; auto strand = strandOfFunction(cb); if (strand) - return strand->schedulerFor(std::move(callGuardedCb)); - return futurizeOutput(std::move(callGuardedCb)); + return strand->schedulerFor(std::move(callSharedCb)); + return futurizeOutput(std::move(callSharedCb)); } void addCallback(Future fut, const ::py::function& cb) diff --git a/src/pyobject.cpp b/src/pyobject.cpp index 4038ed3..9bc1019 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -149,7 +149,7 @@ using GenericFunction = std::function<::py::object(::py::args)>; // @pre `cargs.size() > 0`, arguments must at least contain a `DynamicObject` // on which the function is to be called. AnyReference callPythonMethod(const AnyReferenceVector& cargs, - const GILGuardedObject& method) + const SharedObject<::py::function>& method) { auto it = cargs.begin(); const auto cargsEnd = cargs.end(); @@ -170,7 +170,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, // Convert Python future object into a C++ Future, to allow libqi to unwrap // it. - const ::py::object ret = (*method)(*args); + const ::py::object ret = invokeCatchPythonError(method.inner(), *args); if (::py::isinstance(ret)) return AnyValue::from(ret.cast()).release(); return AnyReference::from(ret).content().clone(); @@ -259,7 +259,7 @@ boost::optional registerMethod(DynamicObjectBuilder& gob, return gob.xAdvertiseMethod(mmb, AnyFunction::fromDynamicFunction( boost::bind(callPythonMethod, _1, - GILGuardedObject(method)))); + SharedObject(method)))); } } // namespace @@ -393,11 +393,7 @@ Object toObject(const ::py::object& obj) // This is a useless callback, needed to keep a reference on the python object. // When the GenericObject is destroyed, the reference is released. - GILGuardedObject guardedObj(obj); - Object anyobj = gob.object([guardedObj](GenericObject*) mutable { - GILAcquire lock; - *guardedObj = {}; - }); + Object anyobj = gob.object([sharedObj = SharedObject(obj)](GenericObject*) {}); // If there was no ObjectUid stored in the python object, store a new one. if (!maybeObjectUid) diff --git a/src/pysignal.cpp b/src/pysignal.cpp index aa3a864..b6c3e9d 100644 --- a/src/pysignal.cpp +++ b/src/pysignal.cpp @@ -26,7 +26,7 @@ namespace constexpr static const auto asyncArgName = "_async"; -AnyReference dynamicCallFunction(const GILGuardedObject& func, +AnyReference dynamicCallFunction(const SharedObject<::py::function>& func, const AnyReferenceVector& args) { GILAcquire lock; @@ -34,7 +34,7 @@ AnyReference dynamicCallFunction(const GILGuardedObject& func, ::py::size_t i = 0; for (const auto& arg : args) pyArgs[i++] = castToPyObject(arg); - (*func)(*pyArgs); + invokeCatchPythonError(func.inner(), *pyArgs); return AnyValue::makeVoid().release(); } @@ -49,7 +49,7 @@ ::py::object connect(F&& connect, SignalSubscriber subscriber(AnyFunction::fromDynamicFunction( boost::bind(dynamicCallFunction, - GILGuardedObject(pyCallback), + SharedObject(pyCallback), _1)), strand.get()); subscriber.setCallType(MetaCallType_Auto); diff --git a/src/pytypes.cpp b/src/pytypes.cpp index 088ba13..fa03102 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -28,16 +28,6 @@ namespace py namespace { -struct ObjectDecRef -{ - ::py::handle obj; - void operator()() const - { - GILAcquire lock; - obj.dec_ref(); - } -}; - template boost::optional<::py::object> tryToCastObjectTo(ObjectTypeInterface* type, void* ptr) diff --git a/tests/test_guard.cpp b/tests/test_guard.cpp index bc16e48..d9d835c 100644 --- a/tests/test_guard.cpp +++ b/tests/test_guard.cpp @@ -39,70 +39,6 @@ TEST_F(InvokeGuardedTest, ExecutesFunctionWithGuard) EXPECT_EQ(52, r); } -using GuardedTest = GuardTest; - -TEST_F(GuardedTest, GuardsConstruction) -{ - using T = GuardedTest; - - struct Check - { - int i; - Check(int i) : i(i) { EXPECT_TRUE(T::guarded); } - explicit operator bool() const { return true; } - }; - - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded g(42); - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, g->i); -} - -TEST_F(GuardedTest, GuardsCopyConstruction) -{ - using T = GuardedTest; - - struct Check - { - int i = 0; - Check(int i) : i(i) {} - Check(const Check& o) - : i(o.i) - { EXPECT_TRUE(T::guarded); } - explicit operator bool() const { return true; } - }; - - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded a(42); - qi::py::detail::Guarded b(a); - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, a->i); - EXPECT_EQ(42, b->i); -} - -TEST_F(GuardedTest, GuardsCopyAssignment) -{ - using T = GuardedTest; - - struct Check - { - int i = 0; - Check(int i) : i(i) {} - Check& operator=(const Check& o) - { EXPECT_TRUE(T::guarded); i = o.i; return *this; } - explicit operator bool() const { return true; } - }; - - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded a(42); - qi::py::detail::Guarded b(37); - EXPECT_FALSE(T::guarded); - b = a; - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, a->i); - EXPECT_EQ(42, b->i); -} - TEST(GILAcquire, IsReentrant) { qi::py::GILAcquire acq0; QI_IGNORE_UNUSED(acq0); @@ -120,3 +56,57 @@ TEST(GILRelease, IsReentrant) qi::py::GILRelease rel2; QI_IGNORE_UNUSED(rel2); SUCCEED(); } + +struct SharedObject : testing::Test +{ + SharedObject() + { + // GIL is only required for creation of the inner object. + object = [&]{ + qi::py::GILAcquire lock; + return pybind11::capsule( + &this->destroyed, + [](void* destroyed){ + *reinterpret_cast(destroyed) = true; + } + ); + }(); + } + + ~SharedObject() + { + if (object) { + qi::py::GILAcquire lock; + object = {}; + } + } + + bool destroyed = false; + pybind11::capsule object; +}; + +TEST_F(SharedObject, KeepsRefCount) +{ + std::optional sharedObject = qi::py::SharedObject(std::move(object)); + ASSERT_FALSE(object); // object has been released. + EXPECT_FALSE(destroyed); // inner object is still alive. + { + auto sharedObjectCpy = *sharedObject; // copy the shared object locally. + EXPECT_FALSE(destroyed); // inner object is maintained by both copies. + sharedObject.reset(); + EXPECT_FALSE(destroyed); // inner object is maintained by the copy. + } + EXPECT_TRUE(destroyed); // inner object has been destroyed. +} + +TEST_F(SharedObject, TakeInnerStealsInnerRefCount) +{ + std::optional sharedObject = qi::py::SharedObject(std::move(object)); + auto inner = sharedObject->takeInner(); + EXPECT_FALSE(sharedObject->inner()); // inner object is null. + sharedObject.reset(); + EXPECT_FALSE(destroyed); // inner object is still alive. + qi::py::GILAcquire lock; // release local inner object, which requires the GIL. + inner = {}; + EXPECT_TRUE(destroyed); // inner object has been destroyed. +} diff --git a/tests/test_qipython_local_interpreter.cpp b/tests/test_qipython_local_interpreter.cpp index c1113f7..581255a 100644 --- a/tests/test_qipython_local_interpreter.cpp +++ b/tests/test_qipython_local_interpreter.cpp @@ -26,13 +26,80 @@ PYBIND11_EMBEDDED_MODULE(test_local_interpreter, m) { })); } -TEST(InterpreterFinalize, DoesNotSegfaultGarbageObjectDtorOutsideGIL) +TEST(InterpreterFinalize, GarbageObjectDtorOutsideGIL) { pybind11::scoped_interpreter interp; pybind11::globals()["qitli"] = pybind11::module::import("test_local_interpreter"); pybind11::exec("obj = qitli.ObjectDtorOutsideGIL()"); } +// This test checks that concurrent uses of GIL guards during finalization +// of the interpreter does not cause crashes. +// +// It uses 2 threads: a main thread, and a second native thread. +// +// The main thread starts an interpreter, starts the second thread and +// releases the GIL. The second thread acquires the GIL, then releases +// it, and waits for the interpreter to finalize. Once it is finalized, +// it tries to reacquire the GIL, and then release it, according to +// GIL guards destructors. +// +// Horizontal lines are synchronization points between the threads. +// +// main thread (T1) +// ~~~~~~~~~~~~~~~~ +// ▼ +// ╔═══════╪════════╗ +// ║ interpreter ║ +// ----------------------------------------------> start native thread T2 +// ║ ╎ ║ native thread (T2) +// ║ ╎ ║ ~~~~~~~~~~~~~~~~~~ +// ║ ╎ ║ ▼ +// ║ ╔═════╪══════╗ ║ ╎ +// ║ ║ GILRelease ║ ║ ╎ -> T1 releases the GIL +// ----------------------------------------------> GIL shift T1 -> T2 +// ║ ║ ╎ ║ ║ ╔═══════╪════════╗ +// ║ ║ ╎ ║ ║ ║ GILAcquire ║ -> T2 acquires the GIL +// ║ ║ ╎ ║ ║ ║ ╔═════╪══════╗ ║ +// ║ ║ ╎ ║ ║ ║ ║ GILRelease ║ ║ -> T2 releases the GIL +// ----------------------------------------------> GIL shift T2 -> T1 +// ║ ╚═════╪══════╝ ║ ║ ║ ╎ ║ ║ -> T1 acquires the GIL +// ╚═══════╪════════╝ ║ ║ ╎ ║ ║ -> interpreter starts finalizing +// ----------------------------------------------> interpreter finalized +// ╎ ║ ╚═════╪══════╝ ║ -> T2 tries to reacquire GIL but fails, it's a noop. +// ╎ ╚═══════╪════════╝ -> T2 tries to release GIL, but it's a noop. +// ╎ ╎ +TEST(InterpreterFinalize, GILReleasedInOtherThread) +{ + // Synchronization mechanism for the first GIL shift, otherwise T1 will release and reacquire the + // GIL instantly without waiting for T2. + qi::Promise shift1Prom; + auto shift1Fut = shift1Prom.future(); + // Second GIL shift does not require an additional synchronization + // mechanism. Waiting for interpreter finalization ensures that the + // GIL was acquired back by thread T1. + // Synchronization mechanism for interpreter finalization. + qi::Promise finalizedProm; + std::future asyncFut; + { + pybind11::scoped_interpreter interp; + asyncFut = std::async( + std::launch::async, + [finalizedFut = finalizedProm.future(), shift1Prom]() mutable { + qi::py::GILAcquire acquire; + // First GIL shift is done. + shift1Prom.setValue(nullptr); + qi::py::GILRelease release; + finalizedFut.value(); + }); + qi::py::GILRelease release; + shift1Fut.value(); + } + finalizedProm.setValue(nullptr); + // Join the thread, wait for the task to finish and ensure no exception was thrown. + asyncFut.get(); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); From a174e5e1488960c7a868fbc78bf68d4a32db400b Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 7 Nov 2023 18:17:31 +0100 Subject: [PATCH 29/40] conan: Uses gtest v1.14. The "cci.20210126" version of the "gtest" package is unmaintained on Conan center. This patch changes the dependency to the latest available package version. --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 9d5e3a5..24b7c68 100644 --- a/conanfile.py +++ b/conanfile.py @@ -59,7 +59,7 @@ class QiPythonConan(ConanFile): ] test_requires = [ - "gtest/cci.20210126", + "gtest/[~1.14]", ] generators = "CMakeToolchain", "CMakeDeps" From 05a04e854dda53bd83288816365b2ecce78a6805 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Tue, 7 Nov 2023 18:19:37 +0100 Subject: [PATCH 30/40] Adds slight improvements to build, conan and CI scripts. --- ...ll.sh => cibuildwheel_linux_before_all.sh} | 17 ++++--------- ci/conan/global.conf | 5 ++++ conanfile.py | 25 ++++++++++--------- pyproject.toml | 4 +-- 4 files changed, 24 insertions(+), 27 deletions(-) rename ci/{cibuildwheel_before_all.sh => cibuildwheel_linux_before_all.sh} (57%) diff --git a/ci/cibuildwheel_before_all.sh b/ci/cibuildwheel_linux_before_all.sh similarity index 57% rename from ci/cibuildwheel_before_all.sh rename to ci/cibuildwheel_linux_before_all.sh index 72ea5e3..a20dcb0 100755 --- a/ci/cibuildwheel_before_all.sh +++ b/ci/cibuildwheel_linux_before_all.sh @@ -12,26 +12,19 @@ yum install -y perl-IPC-Cmd perl-Digest-SHA conan config install "$PACKAGE/ci/conan" # Clone and export libqi to Conan cache. -# Possible improvement: -# Avoid duplicating the version number here with the version -# defined in conanfile.py. +QI_VERSION=$(sed -nE '/^\s*requires\s*=/,/^\s*]/{ s/\s*"qi\/([^"]+)"/\1/p }' "$PACKAGE/conanfile.py") + GIT_SSL_NO_VERIFY=true \ git clone --depth=1 \ - --branch qi-framework-v4.0.1 \ + --branch "qi-framework-v${QI_VERSION}" \ "$LIBQI_REPOSITORY_URL" \ /work/libqi -conan export /work/libqi --version=4.0.1 +conan export /work/libqi --version="${QI_VERSION}" # Install dependencies of libqi-python from Conan, including libqi. -# Only use the build_type as a variable for the build folder name, so -# that the generated CMake preset is named "conan-release". # # Build everything from sources, so that we do not reuse precompiled binaries. # This is because the GLIBC from the manylinux images are often older than the # ones that were used to build the precompiled binaries, which means the binaries # cannot by executed. -conan install "$PACKAGE" \ - --build="*" \ - -c tools.build:skip_test=true \ - -c tools.cmake.cmaketoolchain:generator=Ninja \ - -c tools.cmake.cmake_layout:build_folder_vars=[\"settings.build_type\"] +conan install "$PACKAGE" --build="*" diff --git a/ci/conan/global.conf b/ci/conan/global.conf index bbcc54a..7ad81c5 100644 --- a/ci/conan/global.conf +++ b/ci/conan/global.conf @@ -1,2 +1,7 @@ core:default_profile=default core:default_build_profile=default +tools.build:skip_test=true +tools.cmake.cmaketoolchain:generator=Ninja +# Only use the build_type as a variable for the build folder name, so +# that the generated CMake preset is named "conan-release". +tools.cmake.cmake_layout:build_folder_vars=["settings.build_type"] diff --git a/conanfile.py b/conanfile.py index 24b7c68..bc74ea4 100644 --- a/conanfile.py +++ b/conanfile.py @@ -36,21 +36,22 @@ ] USED_BOOST_COMPONENTS = [ - "atomic", # required by thread - "chrono", # required by thread - "container", # required by thread - "date_time", # required by thread - "exception", # required by thread - "filesystem", # required by libqi - "locale", # required by libqi - "program_options", # required by libqi - "random", # required by libqi - "regex", # required by libqi - "system", # required by thread "thread", + # required by libqi + "filesystem", + "locale", + "program_options", + "random", + "regex", + # required by boost.thread + "atomic", + "chrono", + "container", + "date_time", + "exception", + "system", ] - class QiPythonConan(ConanFile): requires = [ "boost/[~1.78]", diff --git a/pyproject.toml b/pyproject.toml index 2c98a36..3ebf667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,10 +59,8 @@ build-frontend = "build" environment-pass = ["LIBQI_REPOSITORY_URL"] [tool.cibuildwheel.linux] -# Build using the manylinux2014 image manylinux-x86_64-image = "manylinux2014" - -before-all = ["ci/cibuildwheel_before_all.sh {package}"] +before-all = ["ci/cibuildwheel_linux_before_all.sh {package}"] [tool.cibuildwheel.linux.config-settings] "cmake.args" = ["--preset=conan-release"] From 159d04bbb1cf171b8a9554013cffaf7dd5798a31 Mon Sep 17 00:00:00 2001 From: Vincent PALANCHER Date: Mon, 13 Nov 2023 10:55:51 +0000 Subject: [PATCH 31/40] Improve build procedure and document steps in README. This change fixes minor issues with the build system and the CI, and better document build steps and wheel construction in the README. --- CMakeLists.txt | 10 ++- README.rst | 97 ++++++++++++++++++++++++----- ci/cibuildwheel_linux_before_all.sh | 2 +- pyproject.toml | 4 ++ 4 files changed, 94 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ddba9e4..db1acd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,7 +346,10 @@ if(BUILD_TESTING) pybind11::pybind11 GTest::gmock ) - gtest_discover_tests(test_qipython) + gtest_discover_tests( + test_qipython + DISCOVERY_MODE PRE_TEST + ) add_executable(test_qipython_local_interpreter) target_sources( @@ -363,7 +366,10 @@ if(BUILD_TESTING) cxx_standard GTest::gmock ) - gtest_discover_tests(test_qipython_local_interpreter) + gtest_discover_tests( + test_qipython_local_interpreter + DISCOVERY_MODE PRE_TEST + ) endif() if(NOT Python_Interpreter_FOUND) diff --git a/README.rst b/README.rst index 2383130..8d35691 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,30 @@ __ LibQi_repo_ Building ======== -The libqi-python project requires a compiler that supports C++17 to build. +To build the project, you need: -It is built with CMake >= 3.23. +- a compiler that supports C++17. + + - on Ubuntu: `apt-get install build-essential`. + +- CMake with at least version 3.23. + + - on PyPI (**recommended**): `pip install "cmake>=3.23"`. + - on Ubuntu: `apt-get install cmake`. + +- Python with at least version 3.7 and its development libraries. + + - On Ubuntu: `apt-get install libpython3-dev`. + +- a Python `virtualenv`. + + - On Ubuntu: + + .. code-block:: console + + apt-get install python3-venv + python3 -m venv ~/my-venv # Use the path of your convenience. + source ~/my-venv/bin/activate .. note:: The CMake project offers several configuration options and exports a set @@ -21,13 +42,13 @@ It is built with CMake >= 3.23. .. note:: The procedures described below assume that you have downloaded the project - sources and that your current working directory is the project sources root - directory. + sources, that your current working directory is the project sources root + directory and that you are working inside your virtualenv. Conan ^^^^^ -Additionally, libqi-python is available as a Conan 2 project, which means you +Additionally, `libqi-python` is available as a Conan 2 project, which means you can use Conan to fetch dependencies. You can install and/or upgrade Conan 2 and create a default profile in the @@ -46,11 +67,35 @@ Install dependencies from Conan and build with CMake The procedure to build the project using Conan to fetch dependencies is the following. -You must first install the project dependencies in Conan. +Most dependencies are available on Conan Center repository and should not +require any additional steps to make them available. However, you might need to +first get and export the `libqi` recipe into your local Conan cache. .. code-block:: console - conan install . --build=missing -s build_type=Debug + # GitHub is available, but you can also use internal GitLab. + QI_REPOSITORY="https://github.com/aldebaran/libqi.git" + QI_VERSION="4.0.1" # Checkout the version your project need. + QI_PATH="$HOME/libqi" # Or whatever path you want. + git clone \ + --depth=1 `# Only fetch one commit.` \ + --branch "qi-framework-v${QI_VERSION}" \ + "${QI_REPOSITORY}" \ + "${QI_PATH}" + conan export "${QI_PATH}" \ + --version "${QI_VERSION}" # Technically not required but some + # versions of libqi require it + # because of a bug. + +You can then install the `libqi-python` dependencies in Conan. + +.. code-block:: console + + conan install . \ + --build=missing `# Build dependencies binaries that are missing in Conan.` \ + -s build_type=Debug `# Build in debug mode.` \ + -c tools.build:skip_test=true `# Skip tests building for dependencies.` \ + -c '&:tools.build:skip_test=false' # Do not skip tests for the project. This will generate a build directory containing a configuration with a toolchain file that allows CMake to find dependencies inside the Conan cache. @@ -73,11 +118,14 @@ To start building, you need to configure with CMake and then build: cmake --preset conan-linux-x86_64-gcc-debug cmake --build --preset conan-linux-x86_64-gcc-debug -You can then invoke tests using CTest_: +Tests can now be invoked using CTest_, but they require a runtime environment +from Conan so that all dependencies are found: .. code-block:: console + source build/linux-x86_64-gcc-debug/generators/conanrun.sh ctest --preset conan-linux-x86_64-gcc-debug --output-on-failure + source build/linux-x86_64-gcc-debug/generators/deactivate_conanrun.sh Finally, you can install the project in the directory of your choice. @@ -85,8 +133,9 @@ The project defines a single install component, the ``Module`` component. .. code-block:: console - # "cmake --install" does not support preset sadly. - cmake --install build/linux-x86_64-gcc-debug + # `cmake --install` does not support presets sadly. + cmake \ + --install build/linux-x86_64-gcc-debug \ --component Module --prefix ~/my-libqi-python-install Wheel (PEP 517) @@ -101,21 +150,38 @@ dependencies, such as a toolchain generated by Conan: .. code-block:: console - conan install . --build=missing + conan install . \ + --build=missing `# Build dependencies binaries that are missing in Conan.` \ + -c tools.build:skip_test=true # Skip any test. You now can use the ``build`` Python module to build the wheel using PEP 517. .. code-block:: console - export CMAKE_TOOLCHAIN_FILE=$PWD/build/linux-x86_64-gcc-release/generators/conan_toolchain.cmake - python -m build + pip install -U build + python -m build \ + --config-setting cmake.define.CMAKE_TOOLCHAIN_FILE=$PWD/build/linux-x86_64-gcc-release/generators/conan_toolchain.cmake When built that way, the native libraries present in the wheel are most likely incomplete. You will need to use ``auditwheel`` or ``delocate`` to fix it. +.. note:: + `auditwheel` requires the `patchelf` utility program on Linux. You may need + to install it (on Ubuntu: `apt-get install patchelf`). + .. code-block:: console - auditwheel repair --plat manylinux_2_31_x86_64 dist/qi-*.whl + pip install -U auditwheel # or `delocate` on MacOS. + auditwheel repair \ + --strip `# Strip debugging symbols to get a lighter archive.` \ + `# The desired platform, which may differ depending on your build host.` \ + `# With Ubuntu 20.04, we can target manylinux_2_31. Newer versions of` \ + `# Ubuntu will have to target newer versions of manylinux.` \ + `# If you don't need a manylinux archive, you can also target the` \ + `# 'linux_x86_64' platform.` \ + --plat manylinux_2_31_x86_64 \ + `# Path to the wheel archive.` \ + dist/qi-*.whl # The wheel will be by default placed in a `./wheelhouse/` directory. Crosscompiling @@ -123,12 +189,11 @@ Crosscompiling The project supports cross-compiling as explained in the `CMake manual about toolchains`__. You may simply set the ``CMAKE_TOOLCHAIN_FILE`` variable to the -path of the CMake file in your toolchain. +path of the CMake file of your toolchain. __ CMake_toolchains_ .. _LibQi_repo: https://github.com/aldebaran/libqi .. _scikit-build: https://scikit-build.readthedocs.io/en/latest/ -.. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html .. _CMake_toolchains: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html .. _CTest: https://cmake.org/cmake/help/latest/manual/ctest.1.html diff --git a/ci/cibuildwheel_linux_before_all.sh b/ci/cibuildwheel_linux_before_all.sh index a20dcb0..f033f37 100755 --- a/ci/cibuildwheel_linux_before_all.sh +++ b/ci/cibuildwheel_linux_before_all.sh @@ -12,7 +12,7 @@ yum install -y perl-IPC-Cmd perl-Digest-SHA conan config install "$PACKAGE/ci/conan" # Clone and export libqi to Conan cache. -QI_VERSION=$(sed -nE '/^\s*requires\s*=/,/^\s*]/{ s/\s*"qi\/([^"]+)"/\1/p }' "$PACKAGE/conanfile.py") +QI_VERSION=$(sed -nE '/^\s*requires\s*=/,/^\s*]/{ s/\s*"qi\/([^"]+)".*/\1/p }' "$PACKAGE/conanfile.py") GIT_SSL_NO_VERIFY=true \ git clone --depth=1 \ diff --git a/pyproject.toml b/pyproject.toml index 3ebf667..e6957a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,10 @@ repository = "https://github.com/aldebaran/libqi-python" # Rely only on CMake install, not on Python packages detection. wheel.packages = [] +[tool.scikit-build.cmake.define] +# Disable building tests by default when building a wheel. +BUILD_TESTING = "OFF" + [tool.cibuildwheel] build = "cp*manylinux*x86_64" build-frontend = "build" From 38de753f257a3c7bbe0642c67f2b8fb8d75a60fb Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Mon, 4 Dec 2023 11:53:45 +0100 Subject: [PATCH 32/40] conan: Update libqi to v4.0.2. --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index bc74ea4..9e16c4c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -56,7 +56,7 @@ class QiPythonConan(ConanFile): requires = [ "boost/[~1.78]", "pybind11/[^2.9]", - "qi/4.0.1", + "qi/4.0.2", ] test_requires = [ From cd66e1592a3b2a042bc2bacd286d8b5b5ddc9980 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Mon, 4 Dec 2023 12:05:28 +0100 Subject: [PATCH 33/40] Bump version to v3.1.4. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6957a5..794715e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build" [project] name = "qi" description = "LibQi Python bindings" -version = "3.1.4.dev0" +version = "3.1.4" readme = "README.rst" requires-python = ">=3.7" license = { "file" = "COPYING" } From 1dee2509e46e04894b7f67a3f031905200dab26b Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Mon, 4 Dec 2023 16:21:19 +0100 Subject: [PATCH 34/40] Bump version to v3.1.5.dev0. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 794715e..3b7f54a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build" [project] name = "qi" description = "LibQi Python bindings" -version = "3.1.4" +version = "3.1.5.dev0" readme = "README.rst" requires-python = ">=3.7" license = { "file" = "COPYING" } From a7a91d8707174a5248f69fee9dc66ae485622d95 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:00:38 +0200 Subject: [PATCH 35/40] conan: Update dependencies. libqi now defines a `qi_add_module` macro to replace `qi_create_module`. --- CMakeLists.txt | 2 +- conanfile.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index db1acd8..c4dd4a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -297,7 +297,7 @@ if(BUILD_TESTING) include(GoogleTest) find_package(qimodule REQUIRED) - qi_create_module(moduletest NO_INSTALL) + qi_add_module(moduletest) target_sources( moduletest PRIVATE diff --git a/conanfile.py b/conanfile.py index 9e16c4c..d2a74aa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -52,11 +52,12 @@ "system", ] + class QiPythonConan(ConanFile): requires = [ - "boost/[~1.78]", - "pybind11/[^2.9]", - "qi/4.0.2", + "boost/[~1.83]", + "pybind11/[^2.11]", + "qi/[~4]", ] test_requires = [ @@ -77,9 +78,9 @@ class QiPythonConan(ConanFile): # Disable every components of Boost unless we actively use them. default_options.update( { - f"boost/*:without_{_name}": False - if _name in USED_BOOST_COMPONENTS - else True + f"boost/*:without_{_name}": ( + False if _name in USED_BOOST_COMPONENTS else True + ) for _name in BOOST_COMPONENTS } ) From dc28693a5664e2369d901a849ef3aaf4d299af17 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:02:49 +0200 Subject: [PATCH 36/40] types: Fix dict iterator dereference values destruction. #SW-7582 --- src/pytypes.cpp | 36 +++++++++++------------------------- tests/test_types.cpp | 26 +++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/pytypes.cpp b/src/pytypes.cpp index fa03102..41795d3 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -595,20 +595,13 @@ class ListInterface : public ObjectInterfaceBase using DefaultImpl = DefaultTypeImplMethods>; - void* initializeStorage(void* ptr = nullptr) override - { - return DefaultImpl::initializeStorage(ptr); - } - + void* initializeStorage(void* ptr = nullptr) override { return DefaultImpl::initializeStorage(ptr); } void* clone(void* storage) override { return DefaultImpl::clone(storage); } - void destroy(void* storage) override - { - destroyDisownedReferences(storage); - return DefaultImpl::destroy(storage); - } + void destroy(void* storage) override { return DefaultImpl::destroy(storage); } const TypeInfo& info() override { return DefaultImpl::info(); } void* ptrFromStorage(void** s) override { return DefaultImpl::ptrFromStorage(s); } bool less(void* a, void* b) override { return DefaultImpl::less(a, b); } + Iterator* asIterPtr(void** storage) { return static_cast(ptrFromStorage(storage)); } Iterator& asIter(void** storage) { return *asIterPtr(storage); } }; @@ -680,12 +673,13 @@ class DictInterface: public ObjectInterfaceBase std::advance(it, index); const auto key = ::py::reinterpret_borrow<::py::object>(it->first); const auto element = ::py::reinterpret_borrow<::py::object>(it->second); - const auto keyElementPair = std::make_pair(key, element); - auto ref = AnyReference::from(keyElementPair).clone(); + auto keyRef = AnyReference::from(key); + auto elementRef = AnyReference::from(element); + auto pairRef = makeGenericTuple({keyRef, elementRef}); // Store the disowned reference with the list as a context instead of the // iterator because the reference might outlive the iterator. - storeDisownedReference(dictStorage, ref); - return ref; + storeDisownedReference(dictStorage, pairRef); + return pairRef; } void next(void** storage) override { ++asIter(storage).second; } @@ -693,21 +687,13 @@ class DictInterface: public ObjectInterfaceBase using DefaultImpl = DefaultTypeImplMethods>; - void* initializeStorage(void* ptr = nullptr) override - { - return DefaultImpl::initializeStorage(ptr); - } - + void* initializeStorage(void* ptr = nullptr) override { return DefaultImpl::initializeStorage(ptr); } void* clone(void* storage) override { return DefaultImpl::clone(storage); } - void destroy(void* storage) override - { - destroyDisownedReferences(storage); - return DefaultImpl::destroy(storage); - } - + void destroy(void* storage) override { return DefaultImpl::destroy(storage); } const TypeInfo& info() override { return DefaultImpl::info(); } void* ptrFromStorage(void** s) override { return DefaultImpl::ptrFromStorage(s); } bool less(void* a, void* b) override { return DefaultImpl::less(a, b); } + Iterator* asIterPtr(void** storage) { return static_cast(ptrFromStorage(storage)); } Iterator& asIter(void** storage) { return *asIterPtr(storage); } }; diff --git a/tests/test_types.cpp b/tests/test_types.cpp index 5eae486..a3712e1 100644 --- a/tests/test_types.cpp +++ b/tests/test_types.cpp @@ -387,20 +387,40 @@ TEST_F(TypePassing, Recursive) } } +TEST_F(TypePassing, ReverseList) +{ + exec( + "class TestService:\n" + " def func(self, list):\n" + // Test the iterator interface. + " for value in list:\n" + " assert(isinstance(value, str))\n" + " assert(list == ['hello', 'world'])\n" + ); + registerService(); + const std::vector list {"hello", "world"}; + getService().call("func", list); +} + + TEST_F(TypePassing, ReverseDict) { exec( "class TestService:\n" " def func(self, dict):\n" - " return dict == {'one' : 1, 'two' : 2, 'three' : 3}\n" + // Test the iterator interface. + " for key, value in dict.items():\n" + " assert(isinstance(key, str))\n" + " assert(isinstance(value, int))\n" + " assert(dict == {'one' : 1, 'two' : 2, 'three' : 3})\n" ); registerService(); - const std::map expected { + const std::map dict { {"one", 1}, {"two", 2}, {"three", 3}, }; - EXPECT_TRUE(getService().call("func", expected)); + getService().call("func", dict); } TEST_F(TypePassing, LogLevel) From 5641ce74b7dd18a9191ee10936325d5d0bfe4bb1 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:06:05 +0200 Subject: [PATCH 37/40] ci+conan: Detect default and add cppstd17 profiles. --- ci/cibuildwheel_linux_before_all.sh | 3 ++- ci/conan/global.conf | 2 -- ci/conan/profiles/cppstd17 | 2 ++ ci/conan/profiles/default | 8 -------- 4 files changed, 4 insertions(+), 11 deletions(-) create mode 100644 ci/conan/profiles/cppstd17 delete mode 100644 ci/conan/profiles/default diff --git a/ci/cibuildwheel_linux_before_all.sh b/ci/cibuildwheel_linux_before_all.sh index f033f37..199071a 100755 --- a/ci/cibuildwheel_linux_before_all.sh +++ b/ci/cibuildwheel_linux_before_all.sh @@ -9,6 +9,7 @@ pip install 'conan>=2' 'cmake>=3.23' ninja yum install -y perl-IPC-Cmd perl-Digest-SHA # Install Conan configuration. +conan profile detect conan config install "$PACKAGE/ci/conan" # Clone and export libqi to Conan cache. @@ -27,4 +28,4 @@ conan export /work/libqi --version="${QI_VERSION}" # This is because the GLIBC from the manylinux images are often older than the # ones that were used to build the precompiled binaries, which means the binaries # cannot by executed. -conan install "$PACKAGE" --build="*" +conan install "$PACKAGE" --build="*" --profile:all default --profile:all cppstd17 diff --git a/ci/conan/global.conf b/ci/conan/global.conf index 7ad81c5..f90cae0 100644 --- a/ci/conan/global.conf +++ b/ci/conan/global.conf @@ -1,5 +1,3 @@ -core:default_profile=default -core:default_build_profile=default tools.build:skip_test=true tools.cmake.cmaketoolchain:generator=Ninja # Only use the build_type as a variable for the build folder name, so diff --git a/ci/conan/profiles/cppstd17 b/ci/conan/profiles/cppstd17 new file mode 100644 index 0000000..5b1d062 --- /dev/null +++ b/ci/conan/profiles/cppstd17 @@ -0,0 +1,2 @@ +[settings] +compiler.cppstd=gnu17 diff --git a/ci/conan/profiles/default b/ci/conan/profiles/default deleted file mode 100644 index 4ca0f95..0000000 --- a/ci/conan/profiles/default +++ /dev/null @@ -1,8 +0,0 @@ -[settings] -arch=x86_64 -build_type=Release -compiler=gcc -compiler.cppstd=gnu17 -compiler.libcxx=libstdc++ -compiler.version=10 -os=Linux From 9f7f8b77bd3fefe0ceea0daec67ab60da86ce2b6 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:06:57 +0200 Subject: [PATCH 38/40] cibuildwheel: Remove manylinux-x86_64 image specification --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3b7f54a..18de992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ build-frontend = "build" environment-pass = ["LIBQI_REPOSITORY_URL"] [tool.cibuildwheel.linux] -manylinux-x86_64-image = "manylinux2014" before-all = ["ci/cibuildwheel_linux_before_all.sh {package}"] [tool.cibuildwheel.linux.config-settings] From ab137259116d52f8c5f9f91ec9570757681b7608 Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:15:32 +0200 Subject: [PATCH 39/40] ci: Fetch last version of libqi from GitHub. --- .gitlab-ci.yml | 2 -- ci/cibuildwheel_linux_before_all.sh | 10 ++++------ pyproject.toml | 2 -- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61b2b8c..be06278 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,8 +17,6 @@ build-wheel: tags: - docker image: python:3.8 - variables: - LIBQI_REPOSITORY_URL: "https://gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST/qi/libqi" script: - curl -sSL https://get.docker.com/ | sh - pip install cibuildwheel==2.14.1 diff --git a/ci/cibuildwheel_linux_before_all.sh b/ci/cibuildwheel_linux_before_all.sh index 199071a..8a5e5b8 100755 --- a/ci/cibuildwheel_linux_before_all.sh +++ b/ci/cibuildwheel_linux_before_all.sh @@ -13,14 +13,12 @@ conan profile detect conan config install "$PACKAGE/ci/conan" # Clone and export libqi to Conan cache. -QI_VERSION=$(sed -nE '/^\s*requires\s*=/,/^\s*]/{ s/\s*"qi\/([^"]+)".*/\1/p }' "$PACKAGE/conanfile.py") - GIT_SSL_NO_VERIFY=true \ - git clone --depth=1 \ - --branch "qi-framework-v${QI_VERSION}" \ - "$LIBQI_REPOSITORY_URL" \ + git clone \ + --branch master \ + https://github.com/aldebaran/libqi.git \ /work/libqi -conan export /work/libqi --version="${QI_VERSION}" +conan export /work/libqi # Install dependencies of libqi-python from Conan, including libqi. # diff --git a/pyproject.toml b/pyproject.toml index 18de992..429db6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,8 +60,6 @@ BUILD_TESTING = "OFF" build = "cp*manylinux*x86_64" build-frontend = "build" -environment-pass = ["LIBQI_REPOSITORY_URL"] - [tool.cibuildwheel.linux] before-all = ["ci/cibuildwheel_linux_before_all.sh {package}"] From d2c4bbc0c96d24aaf689f6d87a425aadbeac0e2d Mon Sep 17 00:00:00 2001 From: Vincent Palancher Date: Wed, 17 Apr 2024 16:44:28 +0200 Subject: [PATCH 40/40] Bump version to v3.1.5. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 429db6c..6ccd0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build" [project] name = "qi" description = "LibQi Python bindings" -version = "3.1.5.dev0" +version = "3.1.5" readme = "README.rst" requires-python = ">=3.7" license = { "file" = "COPYING" }