diff --git a/.gitignore b/.gitignore index cb55908..136d222 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ build* _skbuild dist -*.egg-info \ No newline at end of file +*.egg-info +CMakeUserPresets.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..be06278 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,53 @@ +stages: + - build + - publish + - release + +.if-tag: &if-tag + if: "$CI_COMMIT_TAG =~ /^qi-python-v/" + +build-wheel: + stage: build + rules: + # Allow manually building on a Git commit on any branch. + - if: $CI_COMMIT_BRANCH + when: manual + # Always build on a Git tag. + - <<: *if-tag + tags: + - docker + image: python:3.8 + script: + - curl -sSL https://get.docker.com/ | sh + - pip install cibuildwheel==2.14.1 + - cibuildwheel --output-dir wheelhouse + artifacts: + paths: + - wheelhouse/ + +publish-wheel: + stage: publish + image: python:latest + rules: + - <<: *if-tag + needs: + - build-wheel + script: + - pip install build twine + - python -m build + - TWINE_PASSWORD="${CI_JOB_TOKEN}" + TWINE_USERNAME=gitlab-ci-token + python -m twine upload + --repository-url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi" + wheelhouse/* + +create-release: + stage: release + rules: + - <<: *if-tag + script: + - echo "Releasing $CI_COMMIT_TAG." + release: + tag_name: $CI_COMMIT_TAG + name: 'lib$CI_COMMIT_TITLE' + description: '$CI_COMMIT_TAG_MESSAGE' diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f3efc..c4dd4a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,39 +1,412 @@ -cmake_minimum_required(VERSION 3.16) +## Copyright (c) 2023 Aldebaran Robotics. All rights reserved. +## Use of this source code is governed by a BSD-style license that can be +## found in the COPYING file. +## +## qi_python CMake project +## ======================= +## +## Parameters: +## ----------- +## BUILD_TESTING +## ~~~~~~~~~~~~~ +## If set to true, enables building the tests. See the documentation of the +## `CTest` module for more details on this variable. + +cmake_minimum_required(VERSION 3.23) if(ANDROID) message(FATAL_ERROR "This project does not support Android, stopping.") endif() -# Find our project CMake files in the dedicated subdirectory. -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - -include(utils) -include(set_globals) +include(cmake/ParsePythonVersion.cmake) -# Read the version of the library directly from the python file and sets it +# Read the version of the library directly from the `pyproject.toml` file and sets it # as the version of the project. -# -# The file is expected to contain a line `__version__ = `. -set(_version_file ${CMAKE_CURRENT_SOURCE_DIR}/qi/_version.py) -file(STRINGS ${_version_file} _version_strings REGEX version) -set(_version_regex "__version__[ \t]*=[ \t]*['\"]([^'\"]+)['\"]") -if(NOT _version_strings MATCHES "${_version_regex}") - message(FATAL_ERROR "Could not find the package version in file '${_version_file}'.") +set(pyproject_file "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml") +file(STRINGS ${pyproject_file} version_strings REGEX version) +set(version_regex "version[ \t]*=[ \t]*\"([^\"]+)\"") +if(NOT version_strings MATCHES "${version_regex}") + message(FATAL_ERROR "Could not find the package version in file '${pyproject_file}'.") +endif() +parse_python_version(QI_PYTHON "${CMAKE_MATCH_1}") + +project(qi_python VERSION "${QI_PYTHON_VERSION_RELEASE}") + +include(cmake/BuildType.cmake) + +# Enable testing with CTest. This defines the BUILD_TESTING option. +# Disable tests by default when cross compiling. +set(build_testing_default TRUE) +if(CMAKE_CROSSCOMPILING) + set(build_testing_default FALSE) endif() -parse_python_version(_version ${CMAKE_MATCH_1}) +option(BUILD_TESTING "Build the testing tree." "${build_testing_default}") +include(CTest) + +if(MSVC) + add_compile_options(/W3) +else() + add_compile_options(-Wall -Wextra) +endif() + +# Output everything directly in predefined directories of the build tree. +# This is required by the SDK layout implementation. +# Also write a "path.conf" file, which is required for executing some tests. +set(sdk_dir "${CMAKE_BINARY_DIR}/sdk") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${sdk_dir}/bin") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${sdk_dir}/lib") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${sdk_dir}/lib") +set(path_conf_dir "${sdk_dir}/share/qi") +set(path_conf_file_path "${path_conf_dir}/path.conf") +if(NOT EXISTS "${path_conf_file_path}") + file(MAKE_DIRECTORY "${path_conf_dir}") + file(TOUCH "${path_conf_file_path}") +endif() + +############################################################################## +# Find Python +############################################################################## +find_package( + Python REQUIRED + COMPONENTS + Development.Module + OPTIONAL_COMPONENTS + Interpreter + Development.Embed +) + +############################################################################## +# Find Boost +############################################################################## +find_package( + Boost REQUIRED + COMPONENTS thread +) + +############################################################################## +# Find Pybind11 +############################################################################## +find_package(pybind11 REQUIRED) + +############################################################################## +# Find LibQi +############################################################################## +find_package(qi REQUIRED) + +############################################################################## +# Convenience library: cxx_standard +############################################################################## +add_library(cxx_standard INTERFACE) + +# The project requires at least C++17. +target_compile_features( + cxx_standard + INTERFACE + cxx_std_17 +) + +############################################################################## +# Library: qi_python_objects +############################################################################## +add_library(qi_python_objects OBJECT) + +target_sources( + qi_python_objects + + PUBLIC + qipython/common.hpp + qipython/common.hxx + qipython/pyapplication.hpp + qipython/pyasync.hpp + qipython/pyclock.hpp + qipython/pyexport.hpp + qipython/pyfuture.hpp + qipython/pylog.hpp + qipython/pymodule.hpp + qipython/pyobject.hpp + qipython/pypath.hpp + qipython/pyproperty.hpp + qipython/pysession.hpp + qipython/pysignal.hpp + qipython/pyguard.hpp + qipython/pytranslator.hpp + qipython/pytypes.hpp + qipython/pystrand.hpp + + PRIVATE + src/pyapplication.cpp + src/pyasync.cpp + src/pyclock.cpp + src/pyexport.cpp + src/pyfuture.cpp + src/pylog.cpp + src/pymodule.cpp + src/pyobject.cpp + src/pypath.cpp + src/pyproperty.cpp + src/pysession.cpp + src/pysignal.cpp + src/pystrand.cpp + src/pytranslator.cpp + src/pytypes.cpp +) + +target_include_directories( + qi_python_objects + PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" +) -project(libqi-python VERSION ${_version_VERSION_RELEASE}) +target_link_libraries( + qi_python_objects + PUBLIC + cxx_standard + Boost::headers + Boost::thread + pybind11::pybind11 + qi::qi +) -option(QIPYTHON_STANDALONE - "If set, builds and installs the module as a standalone, ready to be \ -packaged as a wheel. Otherwise, builds and installs it, ready to be included \ -as part of a system (such as NAOqi) that contains its dependencies." - FALSE) +set_target_properties( + qi_python_objects + PROPERTIES + CXX_EXTENSIONS TRUE + CXX_VISIBILITY_PRESET hidden + # Use PIC as the library is linked into a shared library in the qi_python + # target. + POSITION_INDEPENDENT_CODE TRUE +) -# Including this package creates the common options/flags/functions of qi projects. -find_package(qibuild) +############################################################################## +# Library: qi_python +############################################################################## +pybind11_add_module( + qi_python + # Disable generation of an extension for the shared library. + WITHOUT_SOABI +) -include(set_dependencies) -include(add_targets) -include(set_install_rules) -include(add_tests) +# Generate a Python file containing the version. +set(version_py_file "${CMAKE_CURRENT_BINARY_DIR}/qi/_version.py") +configure_file(qi/_version.py.in "${version_py_file}" @ONLY) + +# Generate a Python file containing information about the native part of the +# module. +set(native_py_file "${CMAKE_CURRENT_BINARY_DIR}/qi/native.py") +if(qi_VERSION) + set(NATIVE_VERSION "${qi_VERSION}") +else() + set(NATIVE_VERSION "unknown") +endif() +configure_file(qi/native.py.in "${native_py_file}" @ONLY) + +set( + qi_python_py_files + qi/__init__.py + qi/logging.py + qi/path.py + qi/translator.py + qi/_binder.py + qi/_type.py +) +set( + qi_python_test_py_files + qi/test/__init__.py + qi/test/conftest.py + qi/test/test_applicationsession.py + qi/test/test_async.py + qi/test/test_call.py + qi/test/test_log.py + qi/test/test_module.py + qi/test/test_promise.py + qi/test/test_property.py + qi/test/test_return_empty_object.py + qi/test/test_session.py + qi/test/test_signal.py + qi/test/test_strand.py + qi/test/test_typespassing.py +) + +target_sources( + qi_python + PUBLIC + ${qi_python_py_files} + PRIVATE + src/module.cpp + ${qi_python_test_py_files} +) + +target_link_libraries( + qi_python + PRIVATE + cxx_standard + qi_python_objects +) + +set_target_properties( + qi_python + PROPERTIES + CXX_EXTENSIONS TRUE + CXX_VISIBILITY_PRESET hidden + LIBRARY_OUTPUT_DIRECTORY qi + RUNTIME_OUTPUT_DIRECTORY qi + INSTALL_RPATH_USE_LINK_PATH TRUE +) + +############################################################################## +# Library: qipython_module_plugin +############################################################################## +add_library(qimodule_python_plugin SHARED) +target_sources( + qimodule_python_plugin + PRIVATE + src/qimodule_python_plugin.cpp +) +target_link_libraries( + qimodule_python_plugin + PRIVATE + cxx_standard + qi::qi +) + +############################################################################## +# Installation +############################################################################## +install( + TARGETS qi_python + LIBRARY DESTINATION qi COMPONENT Module + # On Windows, shared libraries (DLL) are considered `RUNTIME`. + RUNTIME DESTINATION qi COMPONENT Module +) + +# Install Python files. +install( + FILES ${qi_python_py_files} "${version_py_file}" "${native_py_file}" + DESTINATION qi + COMPONENT Module +) + +############################################################################## +# Tests +############################################################################## +if(BUILD_TESTING) + find_package(GTest REQUIRED) + + # Use the CMake GoogleTest module that offers functions to automatically + # discover tests. + include(GoogleTest) + + find_package(qimodule REQUIRED) + qi_add_module(moduletest) + target_sources( + moduletest + PRIVATE + tests/moduletest.cpp + ) + target_link_libraries( + moduletest + PRIVATE + cxx_standard + ) + + add_executable(service_object_holder) + target_sources( + service_object_holder + PRIVATE + tests/service_object_holder.cpp + ) + target_link_libraries( + service_object_holder + PRIVATE + cxx_standard + qi::qi + ) + + if(Python_Development.Embed_FOUND) + add_executable(test_qipython) + target_sources( + test_qipython + PRIVATE + tests/common.hpp + tests/test_qipython.cpp + tests/test_guard.cpp + tests/test_types.cpp + tests/test_signal.cpp + tests/test_property.cpp + tests/test_object.cpp + tests/test_module.cpp + ) + target_link_libraries( + test_qipython + PRIVATE + qi_python_objects + cxx_standard + Python::Python + Boost::headers + pybind11::pybind11 + GTest::gmock + ) + gtest_discover_tests( + test_qipython + DISCOVERY_MODE PRE_TEST + ) + + add_executable(test_qipython_local_interpreter) + target_sources( + test_qipython_local_interpreter + PRIVATE + tests/test_qipython_local_interpreter.cpp + ) + target_link_libraries( + test_qipython_local_interpreter + PRIVATE + Python::Python + pybind11::pybind11 + qi_python_objects + cxx_standard + GTest::gmock + ) + gtest_discover_tests( + test_qipython_local_interpreter + DISCOVERY_MODE PRE_TEST + ) + endif() + + if(NOT Python_Interpreter_FOUND) + message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") + else() + message(STATUS "tests: a compatible Python Interpreter was found, Python tests are enabled.") + + execute_process( + COMMAND "${Python_EXECUTABLE}" -m pytest --version + OUTPUT_QUIET + ERROR_QUIET + RESULT_VARIABLE pytest_version_err + ) + if(pytest_version_err) + message(WARNING "tests: the pytest module does not seem to be installed for this Python Interpreter, " + "the execution of the tests will most likely result in a failure.") + endif() + + # Copy python files so that tests can be run in the build directory, with the generated files. + foreach(file IN LISTS qi_python_py_files qi_python_test_py_files) + configure_file("${file}" "${file}" COPYONLY) + endforeach() + + add_test( + NAME pytest + COMMAND + Python::Interpreter + -m pytest + "${CMAKE_CURRENT_BINARY_DIR}/qi/test" + --exec_path "$ --qi-standalone --qi-listen-url tcp://127.0.0.1:0" + ) + set_tests_properties( + pytest + PROPERTIES + ENVIRONMENT "QI_SDK_PREFIX=${sdk_dir}" + ) + endif() +else() + message(STATUS "tests: tests are disabled") +endif() diff --git a/MANIFEST.in b/MANIFEST.in index 9491f80..cff072a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,8 @@ include README.rst include COPYING +include CMakeLists.txt +graft cmake +graft qi +graft examples +graft qipython +graft src diff --git a/README.rst b/README.rst index 1b65663..8d35691 100644 --- a/README.rst +++ b/README.rst @@ -10,223 +10,190 @@ __ LibQi_repo_ Building ======== -This project supports the building of a *standalone* package (for instance as a -wheel that can be uploaded on PyPi_) or of a "system" archive. +To build the project, you need: -.. _standalone: +- a compiler that supports C++17. -Standalone (wheel) ------------------- + - on Ubuntu: `apt-get install build-essential`. -This build mode is also referred to as the *standalone* mode. It is enabled by -passing ``-DQIPYTHON_STANDALONE=ON`` to the CMake call. The Python setup script -also sets this mode automatically when used. +- CMake with at least version 3.23. -In this mode, the project will build libqi and install all its dependencies as -part of the project. + - on PyPI (**recommended**): `pip install "cmake>=3.23"`. + - on Ubuntu: `apt-get install cmake`. -The package can be built from the ``setup.py`` script: +- Python with at least version 3.7 and its development libraries. -.. code:: bash + - On Ubuntu: `apt-get install libpython3-dev`. - python3 ./setup.py bdist_wheel +- a Python `virtualenv`. -or + - On Ubuntu: -.. code:: bash + .. code-block:: console - pip3 wheel . --no-use-pep517 + apt-get install python3-venv + python3 -m venv ~/my-venv # Use the path of your convenience. + source ~/my-venv/bin/activate -The use of PEP517 is not yet supported and still problematic with our setup -script, so we have to disable it. +.. note:: + The CMake project offers several configuration options and exports a set + of targets when installed. You may refer to the ``CMakeLists.txt`` file + for more details about available parameters and exported targets. -The setup script uses scikit-build_ which is itself based on setuptools_. It -handles any option the latter can handle. Additionally, it can take CMake -arguments, which means that you can almost entirely customize how the native -part is built. +.. note:: + The procedures described below assume that you have downloaded the project + sources, that your current working directory is the project sources root + directory and that you are working inside your virtualenv. -In particular, you can use the ``CMAKE_TOOLCHAIN_FILE`` variable to specify a -toolchain to build the native part of the wheel (e.g. if you are using qi -toolchains): +Conan +^^^^^ -.. code:: bash +Additionally, `libqi-python` is available as a Conan 2 project, which means you +can use Conan to fetch dependencies. - python3 ./setup.py bdist_wheel -DCMAKE_TOOLCHAIN_FILE=$HOME/.local/share/qi/toolchains/my_toolchain/toolchain-my_toolchain.cmake +You can install and/or upgrade Conan 2 and create a default profile in the +following way: -System archive --------------- - -This build mode is also referred to as the "system" mode. This is the default -mode. - -In this mode, CMake will expect libqi to be available as a binary package. The -simplest way to build the package in this mode is to use qibuild, which will -first build libqi then libqi-python. - -.. code:: bash - - qibuild configure - qibuild make - -You can also set the ``QI_DIR`` variable at the CMake call to let it know it of -the location of the libqi package. - -.. code:: bash +.. code-block:: console - mkdir build && cd build - cmake .. -DQI_DIR=/path/to/libqi/install/dir - cmake --build . + # install/upgrade Conan 2 + pip install --upgrade conan~=2 + # create a default profile + conan profile detect +Install dependencies from Conan and build with CMake +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Technical details ------------------ +The procedure to build the project using Conan to fetch dependencies is the +following. -The compiled/native part of this project is based on CMake. It can be built as -any other project of the sort and also through qibuild_, although it requires at -least CMake v3.17. +Most dependencies are available on Conan Center repository and should not +require any additional steps to make them available. However, you might need to +first get and export the `libqi` recipe into your local Conan cache. -Our CMake scripts may take a few parameters: +.. code-block:: console -- ``QIPYTHON_STANDALONE``, when set, builds the library in *standalone* mode. - Refer to the standalone_ section for details. -- ``QIPYTHON_FORCE_STRIP``, when set, forces the build system to strip the - libqi-python native module library at install, resulting in a smaller binary. -- ``QI_WITH_TESTS``, when set, enables building of tests. This option is ignored - when cross-compiling. + # GitHub is available, but you can also use internal GitLab. + QI_REPOSITORY="https://github.com/aldebaran/libqi.git" + QI_VERSION="4.0.1" # Checkout the version your project need. + QI_PATH="$HOME/libqi" # Or whatever path you want. + git clone \ + --depth=1 `# Only fetch one commit.` \ + --branch "qi-framework-v${QI_VERSION}" \ + "${QI_REPOSITORY}" \ + "${QI_PATH}" + conan export "${QI_PATH}" \ + --version "${QI_VERSION}" # Technically not required but some + # versions of libqi require it + # because of a bug. -Dependencies -~~~~~~~~~~~~ +You can then install the `libqi-python` dependencies in Conan. -The project has a few dependencies and the build system might report errors if -it fails to find them. It uses either ``FindXXX`` CMake modules through the -``find_package`` command or the ``FetchContent`` module for subprojects. Either way, -any parameter that these modules and the ``find_package`` command accept can be -used to customize how the build system finds the libraries or fetches the -subprojects. +.. code-block:: console -Most of the variables described here are defined in the -``cmake/set_dependencies.cmake`` file. You may refer to this file for more details -on these variables and their values. + conan install . \ + --build=missing `# Build dependencies binaries that are missing in Conan.` \ + -s build_type=Debug `# Build in debug mode.` \ + -c tools.build:skip_test=true `# Skip tests building for dependencies.` \ + -c '&:tools.build:skip_test=false' # Do not skip tests for the project. -LibQi ->>>>> +This will generate a build directory containing a configuration with a +toolchain file that allows CMake to find dependencies inside the Conan cache. -The project's dependencies on LibQi depends on the building mode: +You can then invoke CMake directly inside the build configuration directory to +configure and build the project. Fortunately, Conan also generates a CMake +preset that simplifies the process. The name of the preset may differ on +your machine. You may need to find the preset generated by Conan first by +calling: -- In **system mode**, it will expect to find it as a binary package. The location - of the binary package installation can be specified through the ``QI_DIR`` - variable. -- In **standalone mode**, it will download and compile it as a subproject through - the ``FetchContent`` CMake module. How it is downloaded can be customized - through the following variables: +.. code-block:: console - - ``LIBQI_VERSION`` - - ``LIBQI_GIT_REPOSITORY`` - - ``LIBQI_GIT_TAG`` + cmake --list-presets -It is possible to skip the download step and use an existing source directory by -setting its path as the ``FETCHCONTENT_SOURCE_DIR_LIBQI`` CMake variable. The -build system will still check that the version of the sources matches the -``LIBQI_VERSION`` value if it is set. +Here, we'll assume that the preset is named `conan-linux-x86_64-gcc-debug`. +To start building, you need to configure with CMake and then build: -Python ->>>>>> +.. code-block:: console -The build system uses the FindPython_ CMake module. It will try to honor the -following variables if they are set: + cmake --preset conan-linux-x86_64-gcc-debug + cmake --build --preset conan-linux-x86_64-gcc-debug - - ``PYTHON_VERSION_STRING`` - - ``PYTHON_LIBRARY`` - - ``PYTHON_INCLUDE_DIR`` +Tests can now be invoked using CTest_, but they require a runtime environment +from Conan so that all dependencies are found: -pybind11 ->>>>>>>> +.. code-block:: console -The build system will by default download and compile pybind11__ as a -subproject through the ``FetchContent`` CMake module. How it is downloaded can be -customized through the following variables: + source build/linux-x86_64-gcc-debug/generators/conanrun.sh + ctest --preset conan-linux-x86_64-gcc-debug --output-on-failure + source build/linux-x86_64-gcc-debug/generators/deactivate_conanrun.sh -__ pybind11_repo_ +Finally, you can install the project in the directory of your choice. - - ``PYBIND11_VERSION`` - - ``PYBIND11_GIT_REPOSITORY`` - - ``PYBIND11_GIT_TAG`` +The project defines a single install component, the ``Module`` component. +.. code-block:: console -Boost ->>>>> + # `cmake --install` does not support presets sadly. + cmake \ + --install build/linux-x86_64-gcc-debug \ + --component Module --prefix ~/my-libqi-python-install -The build system will look for the Boost libraries on the system or in the -toolchain if one is set. The expected version of the libraries is specified as -the ``BOOST_VERSION`` variable. +Wheel (PEP 517) +--------------- -The build system uses the FindBoost_ CMake module. +You may build this project as a wheel package using PEP 517. -OpenSSL ->>>>>>> +It uses a scikit-build_ backend which interfaces with CMake. -The build system uses the FindOpenSSL_ CMake module. +You may need to provide a toolchain file so that CMake finds the required +dependencies, such as a toolchain generated by Conan: -ICU ->>> +.. code-block:: console -The build system uses the FindICU_ CMake module. + conan install . \ + --build=missing `# Build dependencies binaries that are missing in Conan.` \ + -c tools.build:skip_test=true # Skip any test. -GoogleTest ->>>>>>>>>> +You now can use the ``build`` Python module to build the wheel using PEP 517. -The build system will by default download and compile GoogleTest__ as a -subproject through the ``FetchContent`` CMake module. How it is downloaded can be -customized through the following variables: +.. code-block:: console -__ GoogleTest_repo_ + pip install -U build + python -m build \ + --config-setting cmake.define.CMAKE_TOOLCHAIN_FILE=$PWD/build/linux-x86_64-gcc-release/generators/conan_toolchain.cmake - - ``GOOGLETEST_VERSION`` - - ``GOOGLETEST_GIT_REPOSITORY`` - - ``GOOGLETEST_GIT_TAG`` +When built that way, the native libraries present in the wheel are most likely incomplete. +You will need to use ``auditwheel`` or ``delocate`` to fix it. -Install -~~~~~~~ +.. note:: + `auditwheel` requires the `patchelf` utility program on Linux. You may need + to install it (on Ubuntu: `apt-get install patchelf`). -Once the project is configured, it can be built and installed as any CMake -project: - -.. code:: bash - - mkdir build && cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/myinstallpath - cmake --install . +.. code-block:: console + pip install -U auditwheel # or `delocate` on MacOS. + auditwheel repair \ + --strip `# Strip debugging symbols to get a lighter archive.` \ + `# The desired platform, which may differ depending on your build host.` \ + `# With Ubuntu 20.04, we can target manylinux_2_31. Newer versions of` \ + `# Ubuntu will have to target newer versions of manylinux.` \ + `# If you don't need a manylinux archive, you can also target the` \ + `# 'linux_x86_64' platform.` \ + --plat manylinux_2_31_x86_64 \ + `# Path to the wheel archive.` \ + dist/qi-*.whl + # The wheel will be by default placed in a `./wheelhouse/` directory. Crosscompiling -------------- The project supports cross-compiling as explained in the `CMake manual about toolchains`__. You may simply set the ``CMAKE_TOOLCHAIN_FILE`` variable to the -path of the CMake file in your toolchain. +path of the CMake file of your toolchain. __ CMake_toolchains_ -Testing -======= - -When enabled, tests can be executed with `CTest`_. - -.. code:: bash - - cd build - ctest . --output-on-failure - .. _LibQi_repo: https://github.com/aldebaran/libqi -.. _PyPi: https://pypi.org/ .. _scikit-build: https://scikit-build.readthedocs.io/en/latest/ -.. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html -.. _qibuild: https://github.com/aldebaran/qibuild -.. _pybind11_repo: https://pybind11.readthedocs.io/en/latest/ -.. _FindPython: https://cmake.org/cmake/help/latest/module/FindPython.html -.. _FindBoost: https://cmake.org/cmake/help/latest/module/FindBoost.html -.. _FindOpenSSL: https://cmake.org/cmake/help/latest/module/FindOpenSSL.html -.. _FindICU: https://cmake.org/cmake/help/latest/module/FindICU.html -.. _GoogleTest_repo: https://github.com/google/googletest .. _CMake_toolchains: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html .. _CTest: https://cmake.org/cmake/help/latest/manual/ctest.1.html diff --git a/ci/cibuildwheel_linux_before_all.sh b/ci/cibuildwheel_linux_before_all.sh new file mode 100755 index 0000000..8a5e5b8 --- /dev/null +++ b/ci/cibuildwheel_linux_before_all.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -x -e + +PACKAGE=$1 + +pip install 'conan>=2' 'cmake>=3.23' ninja + +# Perl dependencies required to build OpenSSL. +yum install -y perl-IPC-Cmd perl-Digest-SHA + +# Install Conan configuration. +conan profile detect +conan config install "$PACKAGE/ci/conan" + +# Clone and export libqi to Conan cache. +GIT_SSL_NO_VERIFY=true \ + git clone \ + --branch master \ + https://github.com/aldebaran/libqi.git \ + /work/libqi +conan export /work/libqi + +# Install dependencies of libqi-python from Conan, including libqi. +# +# Build everything from sources, so that we do not reuse precompiled binaries. +# This is because the GLIBC from the manylinux images are often older than the +# ones that were used to build the precompiled binaries, which means the binaries +# cannot by executed. +conan install "$PACKAGE" --build="*" --profile:all default --profile:all cppstd17 diff --git a/ci/conan/global.conf b/ci/conan/global.conf new file mode 100644 index 0000000..f90cae0 --- /dev/null +++ b/ci/conan/global.conf @@ -0,0 +1,5 @@ +tools.build:skip_test=true +tools.cmake.cmaketoolchain:generator=Ninja +# Only use the build_type as a variable for the build folder name, so +# that the generated CMake preset is named "conan-release". +tools.cmake.cmake_layout:build_folder_vars=["settings.build_type"] diff --git a/ci/conan/profiles/cppstd17 b/ci/conan/profiles/cppstd17 new file mode 100644 index 0000000..5b1d062 --- /dev/null +++ b/ci/conan/profiles/cppstd17 @@ -0,0 +1,2 @@ +[settings] +compiler.cppstd=gnu17 diff --git a/cmake/BuildType.cmake b/cmake/BuildType.cmake new file mode 100644 index 0000000..7e31d51 --- /dev/null +++ b/cmake/BuildType.cmake @@ -0,0 +1,16 @@ +# Inspired by https://www.kitware.com/cmake-and-the-default-build-type/. +# +# Set a default build type if none was specified. +set(default_build_type "Release") +if(EXISTS "${CMAKE_SOURCE_DIR}/.git") + set(default_build_type "Debug") +endif() + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to '${default_build_type}' as none was specified.") + set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE + STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() diff --git a/cmake/utils.cmake b/cmake/ParsePythonVersion.cmake similarity index 50% rename from cmake/utils.cmake rename to cmake/ParsePythonVersion.cmake index 5db32ab..4f38d1a 100644 --- a/cmake/utils.cmake +++ b/cmake/ParsePythonVersion.cmake @@ -1,5 +1,3 @@ -include_guard(GLOBAL) - # Parses a Python version as defined in PEP440 # (see https://www.python.org/dev/peps/pep-0440/) in some input and sets the # following variables accordingly: @@ -48,59 +46,3 @@ function(parse_python_version prefix input) set(${prefix}_VERSION_RELEASE_MAJOR_MINOR "${CMAKE_MATCH_1}${CMAKE_MATCH_2}" PARENT_SCOPE) set(${prefix}_VERSION_RELEASE_PATCH "${CMAKE_MATCH_5}" PARENT_SCOPE) endfunction() - -# Enables most compiler warnings for a target. -function(enable_warnings target) - if(CMAKE_COMPILER_IS_GNUCC) - target_compile_options(${target} PRIVATE -Wall -Wextra) - elseif(MSVC) - target_compile_options(${target} PRIVATE /W4) - endif() -endfunction() - -# Sets the install RPATH of some targets to a list of paths relative to the -# location of the binary. -function(set_install_rpath) - cmake_parse_arguments(SET_INSTALL_RPATH "ORIGIN" "" "TARGETS;PATHS_REL_TO_ORIGIN" ${ARGN}) - foreach(_target IN LISTS SET_INSTALL_RPATH_TARGETS) - set(_paths) - if(SET_INSTALL_RPATH_ORIGIN) - message(VERBOSE "Adding $ORIGIN to RPATH of ${_target}") - list(APPEND _paths - "$<$:$ORIGIN>$<$:@loader_path>") - endif() - foreach(_path IN LISTS SET_INSTALL_RPATH_PATHS_REL_TO_ORIGIN) - if(NOT _path) - continue() - endif() - message(VERBOSE "Adding $ORIGIN/${_path} to RPATH of ${_target}") - list(APPEND _paths - "$<$:$ORIGIN/${_path}>$<$:@loader_path/${_path}>") - endforeach() - if(_paths) - message(VERBOSE "Setting ${_target} RPATH to ${_paths}") - set_property(TARGET ${_target} PROPERTY INSTALL_RPATH ${_paths}) - endif() - endforeach() -endfunction() - -# Creates a variable that can be overridden by the user from either the -# command-line, the cache or the environment. -# The order of preference is: -# - the value from the variable in cache (or the command-line, since setting -# a variable from the command-line automatically adds it to the cache). -# - the value from the variable in the environment. -# - the default value for the variable. -function(overridable_variable name default_value) - # The variable already exists in the cache. It's the preferred value, so we - # don't change it. - if(DEFINED CACHE{${name}}) - return() - endif() - - set(value ${default_value}) - if(DEFINED ENV{${name}}) - set(value $ENV{${name}}) - endif() - set(${name} ${value} PARENT_SCOPE) -endfunction() diff --git a/cmake/add_targets.cmake b/cmake/add_targets.cmake deleted file mode 100644 index cdb0ae1..0000000 --- a/cmake/add_targets.cmake +++ /dev/null @@ -1,105 +0,0 @@ -include_guard(GLOBAL) - - -add_library(qi_python_objects OBJECT) - -target_sources(qi_python_objects - PUBLIC - qipython/common.hpp - qipython/pyapplication.hpp - qipython/pyasync.hpp - qipython/pyclock.hpp - qipython/pyexport.hpp - qipython/pyfuture.hpp - qipython/pylog.hpp - qipython/pymodule.hpp - qipython/pyobject.hpp - qipython/pypath.hpp - qipython/pyproperty.hpp - qipython/pysession.hpp - qipython/pysignal.hpp - qipython/pyguard.hpp - qipython/pytranslator.hpp - qipython/pytypes.hpp - qipython/pystrand.hpp - - PRIVATE - src/pyapplication.cpp - src/pyasync.cpp - src/pyclock.cpp - src/pyexport.cpp - src/pyfuture.cpp - src/pylog.cpp - src/pymodule.cpp - src/pyobject.cpp - src/pypath.cpp - src/pyproperty.cpp - src/pysession.cpp - src/pysignal.cpp - src/pystrand.cpp - src/pytranslator.cpp - src/pytypes.cpp -) - -enable_warnings(qi_python_objects) - -target_include_directories(qi_python_objects PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - -target_link_libraries(qi_python_objects - PUBLIC cxx11 - pybind11 - qi.interface) - -set_target_properties(qi_python_objects - # Use PIC as the library is linked into a shared library in the qi_python - # target. - PROPERTIES POSITION_INDEPENDENT_CODE TRUE) - - -Python_add_library(qi_python MODULE) - -set(QIPYTHON_PYTHON_MODULE_FILES - qi/__init__.py - qi/logging.py - qi/path.py - qi/translator.py - qi/_binder.py - qi/_type.py - qi/_version.py) - -target_sources(qi_python - PRIVATE src/module.cpp - ${QIPYTHON_PYTHON_MODULE_FILES} - qi/test/__init__.py - qi/test/conftest.py - qi/test/test_applicationsession.py - qi/test/test_async.py - qi/test/test_call.py - qi/test/test_log.py - qi/test/test_module.py - qi/test/test_promise.py - qi/test/test_property.py - qi/test/test_return_empty_object.py - qi/test/test_session.py - qi/test/test_signal.py - qi/test/test_strand.py - qi/test/test_typespassing.py - setup.py - pyproject.toml - README.rst) - -enable_warnings(qi_python) - -target_link_libraries(qi_python PRIVATE cxx11 qi_python_objects) - -set_target_properties(qi_python - PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME} - RUNTIME_OUTPUT_DIRECTORY ${QIPYTHON_PYTHON_MODULE_NAME}) - -set_build_rpath_to_qipython_dependencies(qi_python) - -if(NOT QIPYTHON_STANDALONE) - add_library(qimodule_python_plugin SHARED) - target_sources(qimodule_python_plugin PRIVATE src/qimodule_python_plugin.cpp) - target_link_libraries(qimodule_python_plugin PRIVATE qi.interface) -endif() diff --git a/cmake/add_tests.cmake b/cmake/add_tests.cmake deleted file mode 100644 index ea46f38..0000000 --- a/cmake/add_tests.cmake +++ /dev/null @@ -1,110 +0,0 @@ -include_guard(GLOBAL) - -if(NOT QI_WITH_TESTS) - message(STATUS "tests: tests are disabled") - return() -endif() - -if(CMAKE_CROSSCOMPILING) - message(STATUS "tests: crosscompiling, tests are disabled") - return() -endif() - -enable_testing() - -# Use the CMake GoogleTest module that offers functions to automatically discover -# tests. -include(GoogleTest) - - -find_package(qimodule REQUIRED HINTS ${libqi_SOURCE_DIR}) -qi_create_module(moduletest NO_INSTALL) -target_sources(moduletest PRIVATE tests/moduletest.cpp) -target_link_libraries(moduletest cxx11 qi.interface) -enable_warnings(moduletest) -set_build_rpath_to_qipython_dependencies(moduletest) - - -add_executable(service_object_holder) -target_sources(service_object_holder PRIVATE tests/service_object_holder.cpp) -target_link_libraries(service_object_holder PRIVATE cxx11 qi.interface) -enable_warnings(service_object_holder) -set_build_rpath_to_qipython_dependencies(service_object_holder) - - -add_executable(test_qipython) -target_sources(test_qipython - PRIVATE tests/common.hpp - tests/test_qipython.cpp - tests/test_guard.cpp - tests/test_types.cpp - tests/test_signal.cpp - tests/test_property.cpp - tests/test_object.cpp - tests/test_module.cpp) -target_link_libraries(test_qipython - PRIVATE Python::Python - pybind11 - qi_python_objects - qi.interface - cxx11 - gmock) -# Unfortunately, in some of our toolchains, gtest/gmock headers are found in the qi-framework -# package, which comes first in the include paths order given to the compiler. This causes the -# compiler to use those headers instead of the ones we got from a `FetchContent` of the googletest -# repository. -target_include_directories(test_qipython - BEFORE # Force this path to come first in the list of include paths. - PRIVATE $) -enable_warnings(test_qipython) -set_build_rpath_to_qipython_dependencies(test_qipython) - -set(_sdk_prefix "${CMAKE_BINARY_DIR}/sdk") -gtest_discover_tests(test_qipython EXTRA_ARGS --qi-sdk-prefix ${_sdk_prefix}) - -if(NOT Python_Interpreter_FOUND) - message(WARNING "tests: a compatible Python Interpreter was NOT found, Python tests are DISABLED.") -else() - message(STATUS "tests: a compatible Python Interpreter was found, Python tests are enabled.") - - # qibuild sets these variables which tends to mess up our calls of the Python interpreter, so - # we reset them preemptively. - set(ENV{PYTHONHOME}) - set(ENV{PYTHONPATH}) - - execute_process(COMMAND "${Python_EXECUTABLE}" -m pytest --version - OUTPUT_QUIET ERROR_QUIET - RESULT_VARIABLE pytest_version_err) - if(pytest_version_err) - message(WARNING "tests: the pytest module does not seem to be installed for this Python Interpreter\ -, the execution of the tests will most likely result in a failure.") - endif() - - macro(copy_py_files dir) - file(GLOB _files RELATIVE "${PROJECT_SOURCE_DIR}" CONFIGURE_DEPENDS "${dir}/*.py") - foreach(_file IN LISTS _files) - set_property(DIRECTORY APPEND PROPERTY - CMAKE_CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/${_file}") - file(COPY "${_file}" DESTINATION "${dir}") - endforeach() - endmacro() - - copy_py_files(qi) - copy_py_files(qi/test) - - add_test(NAME pytest - COMMAND Python::Interpreter -m pytest qi/test - --exec_path - "$ --qi-standalone --qi-listen-url tcp://127.0.0.1:0") - - get_filename_component(_ssl_dir "${OPENSSL_SSL_LIBRARY}" DIRECTORY) - set(_pytest_env - "QI_SDK_PREFIX=${_sdk_prefix}" - "LD_LIBRARY_PATH=${_ssl_dir}") - set_tests_properties(pytest PROPERTIES ENVIRONMENT "${_pytest_env}") -endif() - -# Ensure compatibility with qitest by simply running ctest. Timeout is set to 4 minutes. -file(WRITE - "${CMAKE_BINARY_DIR}/qitest.cmake" "--name;run_ctest;--timeout;240;--working-directory;\ -${CMAKE_CURRENT_BINARY_DIR};--env;PYTHONHOME=;--env;PYTHONPATH=;--;${CMAKE_CTEST_COMMAND};-V") diff --git a/cmake/install_runtime_dependencies.cmake b/cmake/install_runtime_dependencies.cmake deleted file mode 100644 index 648e9d3..0000000 --- a/cmake/install_runtime_dependencies.cmake +++ /dev/null @@ -1,94 +0,0 @@ -set(_module_install_dir - "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/${QIPYTHON_PYTHON_MODULE_NAME}") - -# On Windows, shared libraries (DLL) are not in the `lib` subdirectory of -# the dependencies, but in the `bin` directory. So for each dependency directory -# that ends with `/lib`, we add the corresponding `/bin` directory. -set(_runtime_dependencies_dirs ${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}) -if(WIN32) - set(_dep_dirs ${_runtime_dependencies_dirs}) - foreach(_dep_dir IN LISTS _dep_dirs) - if(_dep_dir MATCHES "/lib$") - get_filename_component(_full_dep_bin_dir "${_dep_dir}/../bin" ABSOLUTE) - list(APPEND _runtime_dependencies_dirs "${_full_dep_bin_dir}") - endif() - endforeach() -endif() - -# Install the runtime dependencies of the module, eventually by copying the -# libraries from the toolchain. -message(STATUS "Looking for dependencies of qi_python and qi in: ${_runtime_dependencies_dirs}") -file(GET_RUNTIME_DEPENDENCIES - LIBRARIES "${QIPYTHON_QI_TARGET_FILE}" - MODULES "${QIPYTHON_QI_PYTHON_TARGET_FILE}" - RESOLVED_DEPENDENCIES_VAR _runtime_dependencies - UNRESOLVED_DEPENDENCIES_VAR _unresolved_runtime_dependencies - CONFLICTING_DEPENDENCIES_PREFIX _conflicting_runtime_dependencies - # eay32 is a library that comes with OpenSSL. - PRE_INCLUDE_REGEXES ssl eay crypto icu boost - PRE_EXCLUDE_REGEXES .* - DIRECTORIES ${_runtime_dependencies_dirs}) - -# Processing potential conflicts in dependencies: if there are multiple -# available choices for one of the runtime dependencies, we prefer one -# that is in the toolchain. Otherwise we just fallback to the first -# available path for that dependency. -if(_conflicting_runtime_dependencies_FILENAMES) - message(STATUS "Some conflicts were found for dependencies of qi_python and qi: ${_conflicting_runtime_dependencies_FILENAMES}") - foreach(_filename IN LISTS _conflicting_runtime_dependencies_FILENAMES) - message(STATUS "Conflicts of dependency '${_filename}': ${_conflicting_runtime_dependencies_${_filename}}") - set(_dep) - foreach(_path IN LISTS _conflicting_runtime_dependencies_${_filename}) - if(QIPYTHON_TOOLCHAIN_DIR AND _path MATCHES "^${QIPYTHON_TOOLCHAIN_DIR}") - set(_dep "${_path}") - message(STATUS "Using file '${_dep}' for dependency '${_filename}' as it is in the toolchain.") - break() - endif() - endforeach() - if(NOT _dep) - list(GET _conflicting_runtime_dependencies_${_filename} 0 _dep) - message(STATUS "Fallback to the first available path '${_dep}' for dependency '${_filename}'.") - endif() - list(APPEND _runtime_dependencies "${_dep}") - endforeach() -endif() - -message(STATUS "The dependencies of qi_python and qi are: ${_runtime_dependencies}") -if(_unresolved_runtime_dependencies) - message(STATUS "Some dependencies of qi_python and qi are unresolved: ${_unresolved_runtime_dependencies}") -endif() - -foreach(_needed_dep IN LISTS _runtime_dependencies) - set(_dep "${_needed_dep}") - - # Some of the dependency libraries are symbolic links, and `file(INSTALL)` will - # copy the symbolic link instead of their target (the real library) by default. - # There is an option FOLLOW_SYMLINK_CHAIN that will copy both the library and - # the symlink, but then when we create an archive (a wheel for instance), - # symlinks are replaced by the file they point to. This results in libraries - # being copied multiple times in the same archive with different names. - # - # Therefore we have to detect symlink manually and copy the files ourselves. - while(IS_SYMLINK "${_dep}") - set(_symlink "${_dep}") - file(READ_SYMLINK "${_symlink}" _dep) - if(NOT IS_ABSOLUTE "${_dep}") - get_filename_component(_dir "${_symlink}" DIRECTORY) - set(_dep "${_dir}/${_dep}") - endif() - message(STATUS "Dependency ${_symlink} is a symbolic link, resolved to ${_dep}") - endwhile() - - file(INSTALL "${_dep}" DESTINATION "${_module_install_dir}") - - if(NOT "${_dep}" STREQUAL "${_needed_dep}") - get_filename_component(_dep_filename "${_dep}" NAME) - get_filename_component(_needed_dep_filename "${_needed_dep}" NAME) - set(_installed_dep "${_module_install_dir}/${_dep_filename}") - set(_installed_needed_dep "${_module_install_dir}/${_needed_dep_filename}") - message(STATUS "Renaming ${_installed_dep} into ${_installed_needed_dep}") - file(RENAME "${_installed_dep}" "${_installed_needed_dep}") - list(TRANSFORM CMAKE_INSTALL_MANIFEST_FILES - REPLACE "${_installed_dep}" "${_installed_needed_dep}") - endif() -endforeach() diff --git a/cmake/native.py.in b/cmake/native.py.in deleted file mode 100644 index 351d698..0000000 --- a/cmake/native.py.in +++ /dev/null @@ -1 +0,0 @@ -__version__ = "@NATIVE_VERSION@" \ No newline at end of file diff --git a/cmake/set_dependencies.cmake b/cmake/set_dependencies.cmake deleted file mode 100644 index dab180e..0000000 --- a/cmake/set_dependencies.cmake +++ /dev/null @@ -1,309 +0,0 @@ -# This file finds the dependencies required by the project and creates the -# needed targets. -# -# It can only be included after a call to `project` and only once. -# -# After using CMake standard `find_package` modules, we often set the following -# variables (with the name of a dependency): -# - _PACKAGE_FOUND -# - _LIBRARIES -# - _INCLUDE_DIRS -# -# These variables are the ones the qibuild CMake part uses in its internal -# functions and `find_package` modules (such as `qi_use_lib`). This allows us -# to use CMake standard modules and bypass qibuild own search procedures. -# -# Once this file is loaded, the following targets are expected to be defined: -# -# - Threads::Threads, the thread library. -# - OpenSSL::SSL, aka libssl. -# - OpenSSL::Crypto, aka libcrypto. -# - Python::Interpreter, the Python interpreter, if found. -# - Python::Python, the Python library for embedding the interpreter. -# - Python::Module, the Python library for modules. -# - python_headers, header-only library for Python. -# - Boost::headers, the target for all the header-only libraries. -# - Boost::atomic, aka libboost_atomic. -# - Boost::date_time, aka libboost_date_time. -# - Boost::thread, aka libboost_thread. -# - Boost::chrono, aka libboost_chrono. -# - Boost::filesystem, aka libboost_filesystem. -# - Boost::locale, aka libboost_locale. -# - Boost::regex, aka libboost_regex. -# - Boost::program_options, aka libboost_program_options. -# - Boost::random, aka libboost_random. -# - pybind11, a header-only Python binding library. -# - gtest & gtest_main, the googletest library. -# - gmock & gmock_main, the googlemock library. -# - qi.interface, the high level target for linking with libqi. -# -# When needed, the following targets can also be defined: -# - ICU::i18n, aka libicui18n. -# - ICU::data, aka libicudata. -# - ICU::uc, aka libicuuc. - -include_guard(GLOBAL) - -# Version of Boost to use. It will be used as the argument to the `find_package(Boost)` call. -overridable_variable(BOOST_VERSION 1.64) - -# Version of pybind11 to use. -overridable_variable(PYBIND11_VERSION 2.5.0) - -# URL of the git repository from which to download pybind11. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. -overridable_variable(PYBIND11_GIT_REPOSITORY https://github.com/pybind/pybind11) - -# Git branch name, tag or commit hash to checkout for pybind11. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_TAG` argument. -overridable_variable(PYBIND11_GIT_TAG v${PYBIND11_VERSION}) - -# Version of googletest to use. -overridable_variable(GOOGLETEST_VERSION 1.10.0) - -# URL of the git repository from which to download googletest. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_REPOSITORY` argument. -overridable_variable(GOOGLETEST_GIT_REPOSITORY https://github.com/google/googletest.git) - -# Git branch name, tag or commit hash to checkout for googletest. For more details, see CMake -# `ExternalProject` module documentation of the `GIT_TAG` argument. -overridable_variable(GOOGLETEST_GIT_TAG release-${GOOGLETEST_VERSION}) - -set(PYTHON_VERSION_STRING "" CACHE STRING "Version of Python to look for. This \ -variable can be specified by tools run directly from Python to enforce the \ -use of the same version.") -mark_as_advanced(PYTHON_VERSION_STRING) - -if(NOT COMMAND FetchContent_Declare) - include(FetchContent) -endif() - -# This part is a bit tricky. Some of our internal toolchains tend to confuse the -# CMake standard `find_package` modules, so we try to use some heuristics to -# set hints for these modules. -# -# If we're using a toolchain, it's most likely either one of our internal -# desktop or Yocto toolchain. This should cover most of our building cases, but -# not all of them, especially for users trying to build the library outside of -# the organization. For these, we let them manually specify the needed -# dependencies path. -if(CMAKE_TOOLCHAIN_FILE) - message(VERBOSE "Toolchain file is set.") - get_filename_component(QIPYTHON_TOOLCHAIN_DIR "${CMAKE_TOOLCHAIN_FILE}" DIRECTORY) - get_filename_component(QIPYTHON_TOOLCHAIN_DIR "${QIPYTHON_TOOLCHAIN_DIR}" ABSOLUTE) - message(VERBOSE "Toolchain directory is ${QIPYTHON_TOOLCHAIN_DIR}.") - - # Cross-compiling for Yocto - if(YOCTO_SDK_TARGET_SYSROOT) - message(VERBOSE "Yocto cross compilation detected.") - message(VERBOSE "Yocto target sysroot is '${YOCTO_SDK_TARGET_SYSROOT}'.") - set(_yocto_target_sysroot_usr "${YOCTO_SDK_TARGET_SYSROOT}/usr") - set(_openssl_root ${_yocto_target_sysroot_usr}) - set(_icu_root ${_yocto_target_sysroot_usr}) - set(_boost_root ${_yocto_target_sysroot_usr}) - set(_python_root ${_yocto_target_sysroot_usr}) - - # Probably compiling for desktop - else() - message(VERBOSE "Assuming desktop compilation.") - set(_openssl_root "${QIPYTHON_TOOLCHAIN_DIR}/openssl") - set(_icu_root "${QIPYTHON_TOOLCHAIN_DIR}/icu") - set(_boost_root "${QIPYTHON_TOOLCHAIN_DIR}/boost") - # No definition of _python_root for desktop, we try to use the one - # installed on the system. - endif() - - # These variables are recognized by the associated CMake standard - # `find_package` modules (see their specific documentation for details). - # - # Setting them as cache variable enables user customisation. - set(OPENSSL_ROOT_DIR ${_openssl_root} CACHE PATH "root directory of an OpenSSL installation") - set(ICU_ROOT ${_icu_root} CACHE PATH "the root of the ICU installation") - set(BOOST_ROOT ${_boost_root} CACHE PATH "Boost preferred installation prefix") - - if(_python_root AND EXISTS "${_python_root}") - # The `qibuild` `python3-config.cmake` module must be used to find the - # dependency as it is packaged in our internal toolchains. - # The CMake standard `find_package` module is named `Python3`, looking for - # `PYTHON3` forces CMake to use the module that comes with `qibuild`. - find_package(PYTHON3 - REQUIRED CONFIG - NAMES PYTHON3 Python3 python3 - HINTS ${_python_root}) - - if(PYTHON3_FOUND OR PYTHON3_PACKAGE_FOUND) - # We want to set the `Python_LIBRARY` variable that the FindPython CMake module - # recognizes, in order to instruct it where to find the library (the FindPython - # module search mechanism uses the python-config utility which is not packaged - # well enough in our toolchains for it to work, therefore we have to manually - # tell the module where the library is). - # - # The `qibuild` Python3 module fills the PYTHON3_LIBRARIES variable with the - # format "general;;debug;", which is recognized by - # `target_link_libraries` (see the function documentation for - # details). However, the FindPython CMake module expects an absolute path. - # - # Therefore we create an interface library `Python_interface`, we use - # `target_link_libraries` so that the specific format is parsed, and then - # we get the full path of the library from the `LINK_LIBRARIES` property of - # the target. - add_library(Python_interface INTERFACE) - target_link_libraries(Python_interface INTERFACE ${PYTHON3_LIBRARIES}) - get_target_property(Python_LIBRARY Python_interface INTERFACE_LINK_LIBRARIES) - list(GET PYTHON3_INCLUDE_DIRS 0 Python_INCLUDE_DIR) - endif() - endif() -endif() - - -# C++11 -add_library(cxx11 INTERFACE) -target_compile_features(cxx11 INTERFACE cxx_std_11) - - -# Threads -find_package(Threads REQUIRED) -if(CMAKE_USE_PTHREADS_INIT) - set(PTHREAD_PACKAGE_FOUND TRUE) - set(PTHREAD_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) - set(PTHREAD_INCLUDE_DIRS) -endif() - - -# OpenSSL -find_package(OpenSSL REQUIRED) - - -# Python -parse_python_version(_python_version "${PYTHON_VERSION_STRING}") -if(_python_version_VERSION_STRING) - set(_python_version ${_python_version_VERSION_RELEASE_MAJOR_MINOR}) -else() - set(_python_version 3.5) # default to Python 3.5+. -endif() - -# Set variables that the CMake module recognizes from variables that the -# scikit-build build system module may pass to us. -if(PYTHON_LIBRARY) - set(Python_LIBRARY "${PYTHON_LIBRARY}") -endif() -if(PYTHON_INCLUDE_DIR) - set(Python_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") -endif() - -# First search for Python with Interpreter+Devel components, to allow the module -# to look for the interpreter that goes with the Python development library. -# Then if it could not find both, try looking for the development component -# only, but this time force the module to find it or fail (REQUIRED). -# Some of our toolchains (when crosscompiling for instance) have the Python -# library but not an interpreter that can run on the host. -find_package(Python ${_python_version} COMPONENTS Development Interpreter) -if(NOT Python_FOUND) - find_package(Python ${_python_version} REQUIRED COMPONENTS Development) -endif() - -list(GET Python_LIBRARIES 0 _py_lib) -add_library(python_headers INTERFACE) -target_include_directories(python_headers INTERFACE ${Python_INCLUDE_DIRS}) -if(_py_lib MATCHES "python([23])\\.([0-9]+)([mu]*d[mu]*)") - target_compile_definitions(python_headers INTERFACE Py_DEBUG) -endif() - - -# Boost -set(_boost_comps atomic date_time thread chrono filesystem locale regex - program_options random) -find_package(Boost ${BOOST_VERSION} EXACT REQUIRED COMPONENTS ${_boost_comps}) - -set(BOOST_PACKAGE_FOUND TRUE) -set(BOOST_LIBRARIES ${Boost_LIBRARIES}) -set(BOOST_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) -# Also set the "qibuild variables" for each of the components of boost. -foreach(_comp IN LISTS _boost_comps) - string(TOUPPER ${_comp} _uc_comp) - set(BOOST_${_uc_comp}_PACKAGE_FOUND TRUE) - set(BOOST_${_uc_comp}_LIBRARIES ${Boost_${_uc_comp}_LIBRARY}) - set(BOOST_${_uc_comp}_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) -endforeach() - - -# ICU. These are the components that might be needed by Boost::locale. -find_package(ICU COMPONENTS i18n uc data) -set(ICU_PACKAGE_FOUND ${ICU_FOUND}) -if(ICU_FOUND) - target_link_libraries(Boost::locale INTERFACE ICU::i18n ICU::data ICU::uc) -endif() - - -# pybind11, which is a header-only library. -# It's simpler to just use it as a CMake external project (through FetchContent) -# and download it directly into the build directory, than to try to use a -# preinstalled version through `find_package` -FetchContent_Declare(pybind11 - GIT_REPOSITORY ${PYBIND11_GIT_REPOSITORY} - GIT_TAG ${PYBIND11_GIT_TAG}) -FetchContent_GetProperties(pybind11) -if(NOT pybind11_POPULATED) - FetchContent_Populate(pybind11) - # pybind11 build system and targets do a lot of stuff automatically (such as - # looking for Python libs, adding the C++11 flag, ...). We prefer to do things - # ourselves, so we recreate a target directly instead of adding it as a - # subproject. - add_library(pybind11 INTERFACE) - target_include_directories(pybind11 INTERFACE ${pybind11_SOURCE_DIR}/include) - target_link_libraries(pybind11 INTERFACE cxx11 python_headers) -endif() - - -# googletest -# -# This is the simplest way to depend on googletest & googlemock. As they are -# fairly quick to compile, we can have it as a subproject (as a `FetchContent` -# project). -FetchContent_Declare(googletest - GIT_REPOSITORY ${GOOGLETEST_GIT_REPOSITORY} - GIT_TAG ${GOOGLETEST_GIT_TAG}) -FetchContent_GetProperties(googletest) -if(NOT googletest_POPULATED) - FetchContent_Populate(googletest) - set(INSTALL_GTEST OFF) - add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) -endif() - -set(GTEST_PACKAGE_FOUND TRUE) -set(GMOCK_PACKAGE_FOUND TRUE) -set(GTEST_LIBRARIES $) -set(GMOCK_LIBRARIES $) - -# Unfortunately, we cannot get the `INTERFACE_INCLUDE_DIRECTORIES` of the gtest -# or gmock targets as they might contain generator expressions containing ";" -# characters, which qibuild wrongly splits as elements of a list. Instead, we -# have to build the paths manually. -set(GTEST_INCLUDE_DIRS ${googletest_SOURCE_DIR}/googletest/include) -set(GMOCK_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} - ${googletest_SOURCE_DIR}/googlemock/include) - - -# LibQi -include(set_libqi_dependency) - -# Generate a list of dependency directories that can be used for -# instance to generate RPATH or install those dependencies. -set(QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS ${Boost_LIBRARY_DIRS}) -foreach(_lib IN LISTS OPENSSL_LIBRARIES ICU_LIBRARIES) - if(EXISTS ${_lib}) - get_filename_component(_dir ${_lib} DIRECTORY) - list(APPEND QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS ${_dir}) - endif() -endforeach() -list(REMOVE_DUPLICATES QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS) - -# Adds all the dependency directories as RPATH for a target ouput -# binary in the build directory, so that we may for instance execute -# it directly. -function(set_build_rpath_to_qipython_dependencies target) - message(VERBOSE - "Setting ${target} build RPATH to '${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}'") - set_property(TARGET ${target} PROPERTY - BUILD_RPATH ${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}) -endfunction() diff --git a/cmake/set_globals.cmake b/cmake/set_globals.cmake deleted file mode 100644 index 3bbccd9..0000000 --- a/cmake/set_globals.cmake +++ /dev/null @@ -1,50 +0,0 @@ -include_guard(GLOBAL) - -# Convert relative paths in `target_sources` to absolute. -cmake_policy(SET CMP0076 NEW) - -# Set the default policies for subprojects using an older CMake version -# (specified through `cmake_minimum_required`). Setting the policy directly -# (through `cmake_policy`) is not enough as it will be overwritten by the call -# to `cmake_minimum_required` in the subproject. This is to remove spurious -# warnings we get otherwise. -# -# 0048 - project() command manages VERSION variables. -set(CMAKE_POLICY_DEFAULT_CMP0048 NEW) -# 0056 - Honor link flags in try_compile() source-file signature. -set(CMAKE_POLICY_DEFAULT_CMP0056 NEW) -# 0066 - Honor per-config flags in try_compile() source-file signature. -set(CMAKE_POLICY_DEFAULT_CMP0066 NEW) -# 0060 - Link libraries by full path even in implicit directories. -set(CMAKE_POLICY_DEFAULT_CMP0060 NEW) -# 0063 - Honor visibility properties for all target types. -set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) -# 0074 - find_package uses PackageName_ROOT variables. -set(CMAKE_POLICY_DEFAULT_CMP0074 NEW) -# 0077 - option() honors normal variables. -set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) -# 0082 - Install rules from add_subdirectory() are interleaved with those in caller. -set(CMAKE_POLICY_DEFAULT_CMP0082 NEW) - -# Do not export symbols by default on new targets. -set(CMAKE_CXX_VISIBILITY_PRESET hidden) - -# We use RPATH on systems that handle them. -set(CMAKE_SKIP_RPATH OFF) -set(CMAKE_SKIP_BUILD_RPATH OFF) - -# Do not use the installation RPATH/RUNPATH values for the build directory. -# Our build directories do not necessarily share the same structure as our install -# directories. -set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF) - -# Do not append to the runtime search path (rpath) of installed binaries any -# directory outside the project that is in the linker search path or contains -# linked library files. -# Instead we copy the dependencies we need in the install directory, and we set -# the right RPATH ourselves. -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH OFF) - -set(QIPYTHON_PYTHON_MODULE_NAME qi) -set(QIPYTHON_PYTHON_MODULE_SOURCE_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/${QIPYTHON_PYTHON_MODULE_NAME}) \ No newline at end of file diff --git a/cmake/set_install_rules.cmake b/cmake/set_install_rules.cmake deleted file mode 100644 index 583a400..0000000 --- a/cmake/set_install_rules.cmake +++ /dev/null @@ -1,7 +0,0 @@ -include_guard(GLOBAL) - -if(QIPYTHON_STANDALONE) - include(set_install_rules_standalone) -else() - include(set_install_rules_system) -endif() diff --git a/cmake/set_install_rules_standalone.cmake b/cmake/set_install_rules_standalone.cmake deleted file mode 100644 index e70f3d6..0000000 --- a/cmake/set_install_rules_standalone.cmake +++ /dev/null @@ -1,57 +0,0 @@ -include_guard(GLOBAL) - -if(NOT QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS) - message(AUTHOR_WARNING "The QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS is empty.") -endif() - -# Stripping the binaries enables us to have lighter wheels. -# -# CMake offers a target "install/strip" that normally does that automatically, -# but scikit-build uses the plain "install" target and this behavior is not -# easily customisable. So this option exists so that we may manually strip the -# binaries when set. -option(QIPYTHON_FORCE_STRIP - "If set, forces the stripping of the qi_python module at install." - TRUE) - -if(${QIPYTHON_FORCE_STRIP}) - message(STATUS "The qi_python library will be stripped at install.") - install(CODE "set(CMAKE_INSTALL_DO_STRIP TRUE)" - COMPONENT Module) -endif() - -# qibuild automatically installs a `path.conf` file that we don't really care -# about in our wheel, and we have no other way to disable its installation other -# than letting it be installed then removing it. -install(CODE - "set(_path_conf \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/share/qi/path.conf\") - file(REMOVE \${_path_conf}) - list(REMOVE_ITEM CMAKE_INSTALL_MANIFEST_FILES \${_path_conf})" - COMPONENT runtime) - -if(QIPYTHON_TOOLCHAIN_DIR) - install(CODE "set(QIPYTHON_TOOLCHAIN_DIR \"${QIPYTHON_TOOLCHAIN_DIR}\")" - COMPONENT Module) -endif() - -install(CODE - "set(QIPYTHON_PYTHON_MODULE_NAME \"${QIPYTHON_PYTHON_MODULE_NAME}\") - set(QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS \"${QIPYTHON_BUILD_DEPENDENCIES_LIBRARY_DIRS}\") - set(QIPYTHON_QI_PYTHON_TARGET_FILE \"$\") - set(QIPYTHON_QI_TARGET_FILE \"$\") - set(QIPYTHON_QI_PYTHON_TARGET_FILE_NAME \"$\") - set(QIPYTHON_QI_TARGET_FILE_NAME \"$\")" - COMPONENT Module) -install(SCRIPT cmake/install_runtime_dependencies.cmake COMPONENT Module) - -set_install_rpath(TARGETS qi_python qi ORIGIN) -install(TARGETS qi_python qi - LIBRARY DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - # On Windows, shared libraries (DLL) are considered `RUNTIME`. - RUNTIME DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - COMPONENT Module) - -# Install the Python file containing native informations. -install(FILES "${QIPYTHON_NATIVE_PYTHON_FILE}" - DESTINATION "${QIPYTHON_PYTHON_MODULE_NAME}" - COMPONENT Module) \ No newline at end of file diff --git a/cmake/set_install_rules_system.cmake b/cmake/set_install_rules_system.cmake deleted file mode 100644 index b7d8e73..0000000 --- a/cmake/set_install_rules_system.cmake +++ /dev/null @@ -1,35 +0,0 @@ -include_guard(GLOBAL) - - -if(NOT Python_VERSION_MAJOR OR NOT Python_VERSION_MINOR) - message(AUTHOR_WARNING "The Python major or minor version is unknown.") -endif() -set(_sitelib_dir - "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages/${QIPYTHON_PYTHON_MODULE_NAME}") - -# Set the RPATH of the module to a relative libraries directory. We assume that, -# if the module is installed in "/path/lib/pythonX.Y/site-packages/qi", the -# libraries we need are instead in "/path/lib", so we set the RPATH accordingly. -set_install_rpath(TARGETS qi_python ORIGIN PATHS_REL_TO_ORIGIN "../../..") - -install(TARGETS qi_python - LIBRARY DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -install(FILES ${QIPYTHON_PYTHON_MODULE_FILES} - DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -# Install the Python file containing native information. -install(FILES "${QIPYTHON_NATIVE_PYTHON_FILE}" - DESTINATION "${_sitelib_dir}" - COMPONENT runtime) - -# Set the RPATH of the plugin to a relative libraries directory. We assume that, -# if the plugin is installed in "/path/lib/qi/plugins", the -# libraries we need are instead in "/path/lib", so we set the RPATH accordingly. -set_install_rpath(TARGETS qimodule_python_plugin ORIGIN PATHS_REL_TO_ORIGIN "../..") - -install(TARGETS qimodule_python_plugin - LIBRARY DESTINATION "lib/qi/plugins" - COMPONENT runtime) \ No newline at end of file diff --git a/cmake/set_libqi_dependency.cmake b/cmake/set_libqi_dependency.cmake deleted file mode 100644 index bd32edc..0000000 --- a/cmake/set_libqi_dependency.cmake +++ /dev/null @@ -1,53 +0,0 @@ -overridable_variable(LIBQI_VERSION 2.0.0) - -# Our github clone is sometimes late or is missing tags, so we enable -# customisation of the URL at configuration time, so users can use another clone. -overridable_variable(LIBQI_GIT_REPOSITORY https://github.com/aldebaran/libqi.git) - -overridable_variable(LIBQI_GIT_TAG qi-framework-v${LIBQI_VERSION}) - -if(LIBQI_VERSION) - message(STATUS "LibQi: expected version is \"${LIBQI_VERSION}\"") -endif() - -# Checks that the version present in the `package_xml_file` file is the same as -# the one specified in the `LIBQI_VERSION` cache variable, if it is set. -function(check_libqi_version package_xml_file version_var) - file(STRINGS ${package_xml_file} _package_xml_strings REGEX version) - set(_package_version "") - if(_package_xml_strings MATCHES [[version="([^"]*)"]]) - set(_package_version ${CMAKE_MATCH_1}) - message(STATUS "LibQi: found version \"${_package_version}\" from file '${package_xml_file}'") - if(LIBQI_VERSION AND NOT (LIBQI_VERSION VERSION_EQUAL _package_version)) - message(FATAL_ERROR "LibQi version mismatch (expected \ - \"${LIBQI_VERSION}\", found \"${_package_version}\")") - endif() - else() - set(_msg_type WARNING) - if (LIBQI_VERSION) - set(_msg_type FATAL_ERROR) - endif() - message(${_msg_type} "Could not read LibQi version from file \ -${package_xml_file}. Please check that the file contains a version \ -attribute.") - endif() - set(${version_var} ${_package_version} PARENT_SCOPE) -endfunction() - -if(QIPYTHON_STANDALONE) - include(set_libqi_dependency_standalone) -else() - include(set_libqi_dependency_system) -endif() - -target_link_libraries(qi.interface INTERFACE cxx11) - -# Generate a Python file containing information about the native part of the -# module.\ -set(NATIVE_VERSION "${LIBQI_PACKAGE_VERSION}") -if(NOT NATIVE_VERSION) - set(NATIVE_VERSION "unknown") -endif() -set(QIPYTHON_NATIVE_PYTHON_FILE - "${CMAKE_CURRENT_BINARY_DIR}/${QIPYTHON_PYTHON_MODULE_NAME}/native.py") -configure_file(cmake/native.py.in "${QIPYTHON_NATIVE_PYTHON_FILE}" @ONLY) diff --git a/cmake/set_libqi_dependency_standalone.cmake b/cmake/set_libqi_dependency_standalone.cmake deleted file mode 100644 index a7de9f9..0000000 --- a/cmake/set_libqi_dependency_standalone.cmake +++ /dev/null @@ -1,71 +0,0 @@ -# In standalone mode, LibQi is currently added as a subproject. It's a -# controversial choice but depending on a prebuilt configuration, installation -# or package is messy with our CMake code. It makes compilation longer but it -# ensures consistency. - -include_guard(GLOBAL) - -if(NOT COMMAND FetchContent_Declare) - include(FetchContent) -endif() - -if(FETCHCONTENT_SOURCE_DIR_LIBQI) - message(STATUS "LibQi: using sources at \"${FETCHCONTENT_SOURCE_DIR_LIBQI}\"") -else() - message(STATUS "LibQi: using sources from \"${LIBQI_GIT_REPOSITORY}\" at tag \ -\"${LIBQI_GIT_TAG}\"") -endif() - -FetchContent_Declare(LibQi - GIT_REPOSITORY "${LIBQI_GIT_REPOSITORY}" - GIT_TAG "${LIBQI_GIT_TAG}" -) -FetchContent_GetProperties(LibQi) -if(NOT libqi_POPULATED) - FetchContent_Populate(LibQi) -endif() - -check_libqi_version("${libqi_SOURCE_DIR}/package.xml" LIBQI_PACKAGE_VERSION) - -# Adds LibQi as a subproject which will create a `qi` target. -# -# We use a function in order to scope the value of a few variables only for the -# LibQi subproject. -function(add_libqi_subproject) - # Do not build examples and tests, we don't need them in this project. - set(BUILD_EXAMPLES OFF) - set(QI_WITH_TESTS OFF) - - # Add it as a subproject but use `EXCLUDE_FROM_ALL` to disable the installation - # rules of the subproject files.com - add_subdirectory(${libqi_SOURCE_DIR} ${libqi_BINARY_DIR} EXCLUDE_FROM_ALL) -endfunction() -add_libqi_subproject() - -add_library(qi.interface INTERFACE) - -# Add the subproject include directories as "system" to avoid warnings -# generated by LibQi headers. -target_include_directories(qi.interface SYSTEM INTERFACE - ${libqi_SOURCE_DIR} - ${libqi_BINARY_DIR}/include) - -target_link_libraries(qi.interface INTERFACE - qi - -# qibuild CMake macros fail to propagate LibQi's include directories to the -# interface of the `qi` target, so we have to add them ourselves. For Boost, -# adding the header-only target is sufficient, as it will add the include -# directories, and the link libraries are already part of the `qi` target's -# interface (qibuild propagates those fine). - Boost::boost - -# These targets are not strictly speaking libraries but they add flags that -# disable autolinking in Boost and force dynamic linking, which are required -# on Windows. - Boost::disable_autolinking - Boost::dynamic_linking - -# For the libraries other than Boost, we don't have a header-only target to -# only add the include directories, so we have to directly link to them. - OpenSSL::SSL OpenSSL::Crypto) \ No newline at end of file diff --git a/cmake/set_libqi_dependency_system.cmake b/cmake/set_libqi_dependency_system.cmake deleted file mode 100644 index 46d2168..0000000 --- a/cmake/set_libqi_dependency_system.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# In system mode, LibQi must be accessible through find_package. -find_package(qi REQUIRED CONFIG) - -# If the qi package was found from a build directory in LibQi sources (which -# is the case when built with qibuild), then we expect that `qi_DIR` is set -# to the path `<...>/libqi/build/sdk/cmake`. -# If it was found in one of our toolchains, then we expect that `qi_DIR` is -# set to the path `/libqi(or qi-framework)/share/cmake/qi` in which -# case the `package.xml` file is in `/libqi(or qi-framework)`. -get_filename_component(QIPYTHON_LIBQI_PACKAGE_FILE - "${qi_DIR}/../../../package.xml" ABSOLUTE) -if(NOT EXISTS "${QIPYTHON_LIBQI_PACKAGE_FILE}") - get_filename_component(QIPYTHON_LIBQI_PACKAGE_FILE - "${qi_DIR}/../../package.xml" ABSOLUTE) -endif() - -set(QIPYTHON_LIBQI_PACKAGE_FILE "${QIPYTHON_LIBQI_PACKAGE_FILE}" CACHE FILEPATH - "Path to the `package.xml` file of LibQi.") - -if(LIBQI_VERSION) - if(EXISTS "${QIPYTHON_LIBQI_PACKAGE_FILE}") - check_libqi_version("${QIPYTHON_LIBQI_PACKAGE_FILE}" LIBQI_PACKAGE_VERSION) - else() - message(WARNING "The `package.xml` file for LibQi could not be found at \ -'${QIPYTHON_LIBQI_PACKAGE_FILE}' (the LibQi package was found in '${qi_DIR}'). \ -The version of LibQi could not be verified. Please manually specify a valid \ -path as the 'QIPYTHON_LIBQI_PACKAGE_FILE' variable to fix this issue, or unset \ -the 'LIBQI_VERSION' variable to completely disable this check.") - endif() -endif() - -add_library(qi.interface INTERFACE) -target_include_directories(qi.interface SYSTEM INTERFACE ${QI_INCLUDE_DIRS}) -target_link_libraries(qi.interface INTERFACE ${QI_LIBRARIES}) - -foreach(_dep IN LISTS QI_DEPENDS) - if(NOT ${_dep}_PACKAGE_FOUND) - find_package(${_dep} REQUIRED CONFIG) - endif() - target_include_directories(qi.interface SYSTEM INTERFACE ${${_dep}_INCLUDE_DIRS}) - target_link_libraries(qi.interface INTERFACE ${${_dep}_LIBRARIES}) -endforeach() diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..d2a74aa --- /dev/null +++ b/conanfile.py @@ -0,0 +1,99 @@ +from conan import ConanFile, tools +from conan.tools.cmake import cmake_layout + +BOOST_COMPONENTS = [ + "atomic", + "chrono", + "container", + "context", + "contract", + "coroutine", + "date_time", + "exception", + "fiber", + "filesystem", + "graph", + "graph_parallel", + "iostreams", + "json", + "locale", + "log", + "math", + "mpi", + "nowide", + "program_options", + "python", + "random", + "regex", + "serialization", + "stacktrace", + "system", + "test", + "thread", + "timer", + "type_erasure", + "wave", +] + +USED_BOOST_COMPONENTS = [ + "thread", + # required by libqi + "filesystem", + "locale", + "program_options", + "random", + "regex", + # required by boost.thread + "atomic", + "chrono", + "container", + "date_time", + "exception", + "system", +] + + +class QiPythonConan(ConanFile): + requires = [ + "boost/[~1.83]", + "pybind11/[^2.11]", + "qi/[~4]", + ] + + test_requires = [ + "gtest/[~1.14]", + ] + + generators = "CMakeToolchain", "CMakeDeps" + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + + default_options = { + "boost/*:shared": True, + "openssl/*:shared": True, + "qi/*:with_boost_locale": True, # for `pytranslator.cpp` + } + + # Disable every components of Boost unless we actively use them. + default_options.update( + { + f"boost/*:without_{_name}": ( + False if _name in USED_BOOST_COMPONENTS else True + ) + for _name in BOOST_COMPONENTS + } + ) + + def layout(self): + # Configure the format of the build folder name, based on the value of some variables. + self.folders.build_folder_vars = [ + "settings.os", + "settings.arch", + "settings.compiler", + "settings.build_type", + ] + + # The cmake_layout() sets the folders and cpp attributes to follow the + # structure of a typical CMake project. + cmake_layout(self) diff --git a/examples/authentication_with_application.py b/examples/authentication_with_application.py index 49ac4a7..003b2b2 100644 --- a/examples/authentication_with_application.py +++ b/examples/authentication_with_application.py @@ -28,7 +28,7 @@ def newAuthenticator(self): # Reads a file containing the username on the first line and the password on -# the second line. +# the second line. This is the format used by qilaunch. def read_auth_file(path): with open(path) as f: username = f.readline().strip() @@ -36,27 +36,52 @@ def read_auth_file(path): return (username, password) -def make_application(): - app = qi.Application(sys.argv) - parser = argparse.ArgumentParser() +def make_application(argv=sys.argv): + """ + Create and return the qi.Application, with authentication set up + according to the command line options. + """ + # create the app and edit `argv` in place to remove the consumed + # arguments. + # As a side effect, if "-h" is in the list, it is replaced with "--help". + app = qi.Application(argv) + + # Setup a non-intrusive parser, behaving like `qi.Application`'s own + # parser: + # * don't complain about unknown arguments + # * consume known arguments + # * if the "--help" option is present: + # * print its own options help + # * do not print the main app usage + # * do not call `sys.exit()` + parser = argparse.ArgumentParser(add_help=False, usage=argparse.SUPPRESS) parser.add_argument( "-a", "--authfile", help="Path to the authentication config file. This file must " "contain the username on the first line and the password on the " "second line.") - args = parser.parse_args(sys.argv[1:]) + if "--help" in argv: + parser.print_help() + return app + args, unparsed_args = parser.parse_known_args(argv[1:]) logins = read_auth_file(args.authfile) if args.authfile else ("nao", "nao") factory = AuthenticatorFactory(*logins) app.session.setClientAuthenticatorFactory(factory) + # edit argv in place. + # Note: this might modify sys.argv, like qi.Application does. + argv[1:] = unparsed_args return app if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--msg", default="Hello python") app = make_application() + args = parser.parse_args() logger = qi.Logger("authentication_with_application") logger.info("connecting session") app.start() logger.info("fetching ALTextToSpeech service") tts = app.session.service("ALTextToSpeech") logger.info("Saying something") - tts.call("say", "Hello python") + tts.call("say", args.msg) diff --git a/pyproject.toml b/pyproject.toml index 390d211..6ccd0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,67 @@ +# PEP 518 [build-system] -requires = ["setuptools>=47.1", "wheel>=0.34", "scikit-build>=0.10.1", - "cmake>=3.17", "ninja>=1.5"] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +# PEP 621 +[project] +name = "qi" +description = "LibQi Python bindings" +version = "3.1.5" +readme = "README.rst" +requires-python = ">=3.7" +license = { "file" = "COPYING" } +keywords=[ + "libqi", + "qi", + "naoqi", + "aldebaran", + "robotics", + "robot", + "nao", + "pepper", + "romeo", + "plato", +] +classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Embedded Systems", + "Framework :: Robot Framework :: Library", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", +] +maintainers = [ + { email = "framework@aldebaran.com" }, + { name = "Vincent Palancher", email = "vincent.palancher@aldebaran.com" }, + { name = "Jérémy Monnon", email = "jmonnon@aldebaran.com" }, +] + +[project.urls] +repository = "https://github.com/aldebaran/libqi-python" + +[tool.scikit-build] +# Rely only on CMake install, not on Python packages detection. +wheel.packages = [] + +[tool.scikit-build.cmake.define] +# Disable building tests by default when building a wheel. +BUILD_TESTING = "OFF" + +[tool.cibuildwheel] +build = "cp*manylinux*x86_64" +build-frontend = "build" + +[tool.cibuildwheel.linux] +before-all = ["ci/cibuildwheel_linux_before_all.sh {package}"] + +[tool.cibuildwheel.linux.config-settings] +"cmake.args" = ["--preset=conan-release"] diff --git a/qi/__init__.py b/qi/__init__.py index 9595052..1770583 100644 --- a/qi/__init__.py +++ b/qi/__init__.py @@ -5,11 +5,6 @@ """LibQi Python bindings.""" -import platform -py_version = platform.python_version_tuple() -if py_version < ('3', '5'): - raise RuntimeError('Python version 3.5+ is required.') - import sys # noqa: E402 import atexit # noqa: E402 diff --git a/qi/_version.py b/qi/_version.py deleted file mode 100644 index 4eb28e3..0000000 --- a/qi/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '3.0.0' diff --git a/qi/_version.py.in b/qi/_version.py.in new file mode 100644 index 0000000..11ba4dd --- /dev/null +++ b/qi/_version.py.in @@ -0,0 +1 @@ +__version__ = "@QI_PYTHON_VERSION_STRING@" diff --git a/qi/native.py.in b/qi/native.py.in new file mode 100644 index 0000000..8a158e2 --- /dev/null +++ b/qi/native.py.in @@ -0,0 +1 @@ +__version__ = "@NATIVE_VERSION@" diff --git a/qi/test/test_module.py b/qi/test/test_module.py index eb41e39..b6564e4 100644 --- a/qi/test/test_module.py +++ b/qi/test/test_module.py @@ -41,3 +41,55 @@ def test_module_service(): cat = session.service("Cat") assert cat.meow(3) == 'meow' + +def test_module_service_object_lifetime(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + cat = session.service("Cat") + + # We use `del` to release the reference to an object, but there is no + # guarantee that the interpreter will finalize the object right away. + # In CPython, this is "mostly" guaranteed as long as it is the last + # reference to the object and that there is no cyclic dependency of + # reference anywhere that might hold the reference. + # As there is no practical alternative, we consider that assuming the + # object is finalized immediately is an acceptable hypothesis. + + # Purr has a property, Play has a signal, Sleep has none. + sleep = cat.makeSleep() + assert cat.nbSleep() == 1 + del sleep + assert cat.nbSleep() == 0 + + purr = cat.makePurr() + assert cat.nbPurr() == 1 + del purr + assert cat.nbPurr() == 0 + + play = cat.makePlay() + assert cat.nbPlay() == 1 + del play + assert cat.nbPlay() == 0 + +def test_object_bound_functions_arguments_conversion_does_not_leak(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + cat = session.service("Cat") + + play = cat.makePlay() + assert cat.nbPlay() == 1 + cat.order(play) + assert cat.nbPlay() == 1 + del play + assert cat.nbPlay() == 0 + +def test_temporary_object_bound_properties_are_usable(): + session = qi.Session() + session.listenStandalone("tcp://localhost:0") + session.loadServiceRename("moduletest.Cat", "", "truc") + # The `volume` member is a bound property of the `Purr` object. + # It should keep the object alive so that setting the property, which + # requires accessing the object, does not fail. + session.service("Cat").makePurr().volume.setValue(42) diff --git a/qiproject.xml b/qiproject.xml deleted file mode 100644 index b106649..0000000 --- a/qiproject.xml +++ /dev/null @@ -1,13 +0,0 @@ - - Joël Lamotte - Jérémy Monnon - Philippe Martin - Vincent Palancher - Julien Bernard - - - - - - - diff --git a/qipython.pml b/qipython.pml deleted file mode 100644 index b55798b..0000000 --- a/qipython.pml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/qipython/common.hpp b/qipython/common.hpp index f9b99e2..3335f66 100644 --- a/qipython/common.hpp +++ b/qipython/common.hpp @@ -19,6 +19,23 @@ #include #include + +// MAJOR, MINOR and PATCH must be in [0, 255]. +#define QI_PYBIND11_VERSION(MAJOR, MINOR, PATCH) \ + (((MAJOR) << 16) | \ + ((MINOR) << 8) | \ + ((PATCH) << 0)) + +#ifdef PYBIND11_VERSION_HEX +// Remove the lowest 8 bits which represent the serial and level version components. +# define QI_CURRENT_PYBIND11_VERSION (PYBIND11_VERSION_HEX >> 8) +#else +# define QI_CURRENT_PYBIND11_VERSION \ + QI_PYBIND11_VERSION(PYBIND11_VERSION_MAJOR, \ + PYBIND11_VERSION_MINOR, \ + PYBIND11_VERSION_PATCH) +#endif + namespace qi { namespace py @@ -93,30 +110,40 @@ boost::optional extractKeywordArg(pybind11::dict kwargs, return pyArg.cast(); } -/// A deleter that deletes the pointer outside of the GIL. -/// -/// Useful for types that might deadlock on destruction if they keep the GIL -/// locked. -struct DeleteOutsideGIL +/// Returns whether or not the interpreter is finalizing. If the information +/// could not be fetched, the function returns an empty optional. Otherwise, it +/// returns an optional set with a boolean stating if the interpreter is indeed +/// finalizing or not. +inline boost::optional interpreterIsFinalizing() { - template - void operator()(T* ptr) const - { - pybind11::gil_scoped_release unlock; - delete ptr; - } -}; +// `_Py_IsFinalizing` is only available on CPython 3.7+ +#if PY_VERSION_HEX >= 0x03070000 + return boost::make_optional(_Py_IsFinalizing() != 0); +#else + // There is no way of knowing on older versions. + return {}; +#endif +} + +/// Acquires the GIL, then invokes a function with the given arguments, but +/// catches any `pybind11::error_already_set` exception thrown during this +/// invocation, and throws a `std::runtime_error` instead. This is useful to +/// avoid the undefined behavior caused by the `what` member function of +/// `pybind11::error_already_set` when the GIL is not acquired. +template +auto invokeCatchPythonError(F&& f, Args&&... args); -/// Delay the destruction of an object to a thread. -struct DeleteInOtherThread +/// Exception type thrown when an operation fails because the interpreter is +/// finalizing. +/// +/// Some operations are forbidden during interpreter finalization as they may +/// terminate the thread they are called from. This exception allows stopping +/// the flow of execution in this case before such functions are called. +struct InterpreterFinalizingException : std::exception { - template - void operator()(T* ptr) const + inline const char* what() const noexcept override { - pybind11::gil_scoped_release unlock; - auto fut = std::async(std::launch::async, [](std::unique_ptr) {}, - std::unique_ptr(ptr)); - QI_IGNORE_UNUSED(fut); + return "the interpreter is finalizing"; } }; @@ -198,4 +225,6 @@ namespace detail PYBIND11_DECLARE_HOLDER_TYPE(T, boost::shared_ptr); +#include + #endif // QIPYTHON_COMMON_HPP diff --git a/qipython/common.hxx b/qipython/common.hxx new file mode 100644 index 0000000..2618b47 --- /dev/null +++ b/qipython/common.hxx @@ -0,0 +1,37 @@ +/* +** Copyright (C) 2023 Aldebaran Robotics +** See COPYING for the license +*/ + +#pragma once + +#ifndef QIPYTHON_COMMON_HXX +#define QIPYTHON_COMMON_HXX + +#include +#include +#include + +namespace qi +{ +namespace py +{ + +template +auto invokeCatchPythonError(F&& f, Args&&... args) +{ + GILAcquire acquire; + try + { + return std::invoke(std::forward(f), std::forward(args)...); + } + catch (const pybind11::error_already_set& err) + { + throw std::runtime_error(err.what()); + } +} + +} // namespace py +} // namespace qi + +#endif // QIPYTHON_COMMON_HXX diff --git a/qipython/pyfuture.hpp b/qipython/pyfuture.hpp index 85b810f..82468e2 100644 --- a/qipython/pyfuture.hpp +++ b/qipython/pyfuture.hpp @@ -25,12 +25,12 @@ using Promise = qi::Promise; // future or its value. inline pybind11::object resultObject(const Future& fut, bool async) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; if (async) return castToPyObject(fut); // Wait for the future outside of the GIL. - auto res = invokeGuarded(qi::SrcFuture{}, fut); + auto res = invokeGuarded(qi::SrcFuture{}, fut); return castToPyObject(res); } diff --git a/qipython/pyguard.hpp b/qipython/pyguard.hpp index 2bc42c4..f20ee1b 100644 --- a/qipython/pyguard.hpp +++ b/qipython/pyguard.hpp @@ -10,6 +10,7 @@ #include #include +#include #include namespace qi @@ -17,6 +18,27 @@ namespace qi namespace py { +/// Returns whether or not the GIL is currently held by the current thread. If +/// the interpreter is not yet initialized or has been finalized, returns an +/// empty optional, as there is no GIL available. +inline boost::optional currentThreadHoldsGil() +{ + // PyGILState_Check() returns 1 (success) before the creation of the GIL and + // after the destruction of the GIL. + if (Py_IsInitialized() == 1) { + const auto gilAcquired = PyGILState_Check() == 1; + return boost::make_optional(gilAcquired); + } + return boost::none; +} + +/// Returns whether or not the GIL exists (i.e the interpreter is initialized +/// and not finalizing) and is currently held by the current thread. +inline bool gilExistsAndCurrentThreadHoldsIt() +{ + return currentThreadHoldsGil().value_or(false); +} + /// DefaultConstructible Guard /// Procedure<_ (Args)> F template @@ -29,105 +51,211 @@ ka::ResultOf invokeGuarded(F&& f, Args&&... args) namespace detail { -// Guards the copy, construction and destruction of an object, and only when -// in case of copy construction the source object evaluates to true, and in case -// of destruction, the object itself evaluates to true. -// -// Explanation: -// pybind11::object only requires the GIL to be locked on copy and if the -// source is not null. Default construction does not require the GIL either. -template -class Guarded +/// G == pybind11::gil_scoped_acquire || G == pybind11::gil_scoped_release +template +void pybind11GuardDisarm(G& guard) { - using Storage = - typename std::aligned_storage::type; - Storage _object; + QI_IGNORE_UNUSED(guard); +// The disarm API was introduced in v2.6.2. +#if QI_CURRENT_PYBIND11_VERSION >= QI_PYBIND11_VERSION(2,6,2) + guard.disarm(); +#endif +} - Object& object() { return reinterpret_cast(_object); } - const Object& object() const { return reinterpret_cast(_object); } +} // namespace detail -public: - template::value, int> = 0> - explicit Guarded(Arg0&& arg0, Args&&... args) +/// RAII utility type that guarantees that the GIL is locked for the scope of +/// the lifetime of the object. If the GIL cannot be acquired (for example, +/// because the interpreter is finalizing), throws an `InterpreterFinalizingException` +/// exception. +/// +/// Objects of this type (or objects composed of them) must not be kept alive +/// after the hand is given back to the interpreter. +/// +/// This type is re-entrant. +/// +/// postcondition: `GILAcquire acq;` establishes `gilExistsAndCurrentThreadHoldsIt()` +struct GILAcquire +{ + inline GILAcquire() { - Guard g; - new(&_object) Object(std::forward(arg0), std::forward(args)...); - } + if (gilExistsAndCurrentThreadHoldsIt()) + return; - Guarded() - { - // Default construction, no GIL needed. - new(&_object) Object(); - } + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (isFinalizing) + throw InterpreterFinalizingException(); - Guarded(Guarded& o) - { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - new(&_object) Object(o.object()); + _state = ::PyGILState_Ensure(); + QI_ASSERT(gilExistsAndCurrentThreadHoldsIt()); } - Guarded(const Guarded& o) + inline ~GILAcquire() { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - new(&_object) Object(o.object()); + // Even if releasing the GIL while the interpreter is finalizing is allowed, it does + // require the GIL to be currently held. But we have no guarantee that this is the case, + // because the GIL may have been released since we acquired it, and we could not + // reacquire it after that (maybe the interpreter is in fact finalizing). + // Therefore, only release the GIL if it is currently held by this thread. + if (_state && gilExistsAndCurrentThreadHoldsIt()) + ::PyGILState_Release(*_state); } - Guarded(Guarded&& o) + + GILAcquire(const GILAcquire&) = delete; + GILAcquire& operator=(const GILAcquire&) = delete; + +private: + boost::optional _state; +}; + +/// RAII utility type that (as a best effort) tries to ensure that the GIL is +/// unlocked for the scope of the lifetime of the object. +/// +/// Objects of this type (or objects composed of them) must not be kept alive +/// after the hand is given back to the interpreter. +/// +/// This type is re-entrant. +/// +/// postcondition: `GILRelease rel;` establishes +/// `interpreterIsFinalizing().value_or(false) || +/// !gilExistsAndCurrentThreadHoldsIt()` +struct GILRelease +{ + inline GILRelease() { - new(&_object) Object(std::move(o.object())); + // Even if releasing the GIL while the interpreter is finalizing is allowed, + // it does require the GIL to be currently held. However, reacquiring the + // GIL is forbidden if finalization started. + // + // It may happen that we try to release the GIL from a function ('F') called + // by the Python interpreter, while it is holding the GIL. In this case, if + // we try to release it, then fail to reacquire it and return the hand to + // the interpreter, the interpreter will most likely terminate the process, + // as it expects to still be holding the GIL. Failure to reacquire the GIL + // can only happen if the interpreter started finalization, either before we + // released it, or while it was released. + // + // Fortunately, in this case, because the interpreter is busy executing the + // function 'F', it is not possible for it to start finalization until 'F' + // returns. This means that the only possible reason of failure of + // reacquisition of the GIL is because the interpreter was finalizing + // *before* calling 'F', therefore before we released to GIL. Therefore, to + // prevent this failure, we check if the interpreter is finalizing before + // releasing the GIL, and if it is, we do nothing, and the GIL stays held. + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (!isFinalizing && gilExistsAndCurrentThreadHoldsIt()) + _release.emplace(); + QI_ASSERT(isFinalizing || !gilExistsAndCurrentThreadHoldsIt()); } - ~Guarded() + inline ~GILRelease() { - boost::optional optGuard; - if (object()) - optGuard.emplace(); - object().~Object(); + // Reacquiring the GIL is forbidden when the interpreter is finalizing, as + // it may terminate the current thread. + const auto isFinalizing = interpreterIsFinalizing().value_or(false); + if (_release && isFinalizing) + detail::pybind11GuardDisarm(*_release); } - Guarded& operator=(const Guarded& o) + GILRelease(const GILRelease&) = delete; + GILRelease& operator=(const GILRelease&) = delete; + +private: + boost::optional _release; +}; + +/// Wraps a Python object as a shared reference-counted value that does not +/// require the GIL to copy, move or assign to. +/// +/// On destruction, it releases the object with the GIL acquired. However, if +/// the GIL cannot be acquired, the object is leaked. This is most likely +/// mitigated by the fact that if the GIL is not available, it means the object +/// already has been or soon will be garbage collected by interpreter +/// finalization. +template +class SharedObject +{ + static_assert(std::is_base_of_v, + "template parameter T must be a subclass of pybind11::object"); + + struct State { - if (this == &o) - return *this; + std::mutex mutex; + T object; + }; + std::shared_ptr _state; + struct StateDeleter + { + inline void operator()(State* state) const { - boost::optional optGuard; - if (o.object()) - optGuard.emplace(); - object() = o.object(); + auto handle = state->object.release(); + delete state; + + // Do not lock the GIL if there is nothing to release. + if (!handle) + return; + + try + { + GILAcquire acquire; + handle.dec_ref(); + } + catch (const qi::py::InterpreterFinalizingException&) + { + // Nothing, the interpreter is finalizing. + } } - return *this; - } + }; - Guarded& operator=(Guarded&& o) +public: + SharedObject() = default; + + inline explicit SharedObject(T object) + : _state(new State { std::mutex(), std::move(object) }, StateDeleter()) { - if (this == &o) - return *this; + } - object() = std::move(o.object()); - return *this; + /// Copies the inner Python object value by incrementing its reference count. + /// + /// @pre: If the inner value is not null, the GIL must be acquired. + T inner() const + { + QI_ASSERT_NOT_NULL(_state); + std::scoped_lock lock(_state->mutex); + return _state->object; } - Object& operator*() { return object(); } - const Object& operator*() const { return object(); } - Object* operator->() { return &object(); } - const Object* operator->() const { return &object(); } - explicit operator Object&() { return object(); } - explicit operator const Object&() const { return object(); } + /// Takes the inner Python object value and leaves a null value in its place. + /// + /// Any copy of the shared object will now have a null inner value. + /// + /// This operation does not require the GIL as the reference count of the + /// object is preserved. + T takeInner() + { + QI_ASSERT_NOT_NULL(_state); + std::scoped_lock lock(_state->mutex); + /// Hypothesis: Moving a `pybind11::object` steals the object and preserves + /// its reference count and therefore does not require the GIL. + return ka::exchange(_state->object, {}); + } }; -} // namespace detail - -/// Wraps a pybind11::object value and locks the GIL on copy and destruction. +/// Deleter that deletes the pointer outside the GIL. /// -/// This is useful for instance to put pybind11 objects in lambda functions so -/// that they can be copied around safely. -using GILGuardedObject = detail::Guarded; +/// Useful for types that might deadlock on destruction if they keep the GIL +/// locked. +struct DeleteOutsideGIL +{ + template + void operator()(T* ptr) const + { + GILRelease unlock; + delete ptr; + } +}; } // namespace py } // namespace qi diff --git a/setup.py b/setup.py deleted file mode 100755 index 5718b87..0000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import platform -from setuptools import find_packages -from skbuild import setup - -py_version = platform.python_version_tuple() -if py_version < ('3', '5'): - raise RuntimeError('Python 3.5+ is required.') - -here = os.path.abspath(os.path.dirname(__file__)) - -version = {} -with open(os.path.join(here, "qi", "_version.py")) as f: - exec(f.read(), version) - -# Get the long description from the README file -with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='qi', - version=version['__version__'], - description='LibQi Python bindings', - long_description=long_description, - long_description_content_type='text/x-rst', - keywords=['libqi', 'qi', 'naoqi', - 'softbank', 'robotics', 'aldebaran', - 'robot', 'nao', 'pepper', 'romeo'], - url='https://github.com/aldebaran/libqi-python', - author='SoftBank Robotics Europe', - author_email='release@softbankrobotics.com', - license='BSD 3-Clause License', - python_requires='~=3.5', - packages=find_packages(exclude=[ - '*.test', '*.test.*', 'test.*', 'test' - ]), - include_package_data=True, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Application Frameworks', - 'Topic :: Software Development :: Embedded Systems', - 'Framework :: Robot Framework :: Library', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3 :: Only', - ], - cmake_args=['-DQIPYTHON_STANDALONE:BOOL=ON'], -) diff --git a/src/pyapplication.cpp b/src/pyapplication.cpp index 48d8052..e9a0ed0 100644 --- a/src/pyapplication.cpp +++ b/src/pyapplication.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -80,25 +81,24 @@ void exportApplication(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; - class_>( + class_( m, "Application") .def(init(withArgcArgv<>([](int& argc, char**& argv) { - gil_scoped_release unlock; + GILRelease unlock; return new Application(argc, argv); })), "args"_a) - .def_static("run", &Application::run, call_guard()) - .def_static("stop", &Application::stop, call_guard()); + .def_static("run", &Application::run, call_guard()) + .def_static("stop", &Application::stop, call_guard()); - class_>( + class_( m, "ApplicationSession") .def(init(withArgcArgv( [](int& argc, char**& argv, bool autoExit, const std::string& url) { - gil_scoped_release unlock; + GILRelease unlock; ApplicationSession::Config config; if (!autoExit) config.setOption(qi::ApplicationSession::Option_NoAutoExit); @@ -108,22 +108,22 @@ void exportApplication(::py::module& m) })), "args"_a, "autoExit"_a, "url"_a) - .def("run", &ApplicationSession::run, call_guard(), + .def("run", &ApplicationSession::run, call_guard(), doc("Block until the end of the program (call " ":py:func:`qi.ApplicationSession.stop` to end the program).")) .def_static("stop", &ApplicationSession::stop, - call_guard(), + call_guard(), doc( "Ask the application to stop, the run function will return.")) .def("start", &ApplicationSession::startSession, - call_guard(), + call_guard(), doc("Start the connection of the session, once this function is " "called everything is fully initialized and working.")) .def_static("atRun", &ApplicationSession::atRun, - call_guard(), "func"_a, + call_guard(), "func"_a, doc( "Add a callback that will be executed when run() is called.")) @@ -131,7 +131,7 @@ void exportApplication(::py::module& m) [](const ApplicationSession& app) { return app.url().str(); }, - call_guard(), + call_guard(), doc("The url given to the Application. It's the url " "used to connect the session.")) diff --git a/src/pyasync.cpp b/src/pyasync.cpp index b66fdd8..f39ca2e 100644 --- a/src/pyasync.cpp +++ b/src/pyasync.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -29,7 +30,7 @@ ::py::object async(::py::function pyCallback, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; qi::uint64_t usDelay = 0; if (const auto optUsDelay = extractKeywordArg(kwargs, delayArgName)) @@ -37,19 +38,15 @@ ::py::object async(::py::function pyCallback, const MicroSeconds delay(usDelay); - GILGuardedObject guardCb(pyCallback); - GILGuardedObject guardArgs(std::move(args)); - GILGuardedObject guardKwargs(std::move(kwargs)); - + SharedObject sharedCb(pyCallback); + SharedObject sharedArgs(std::move(args)); + SharedObject sharedKwArgs(std::move(kwargs)); auto invokeCallback = [=]() mutable { - ::py::gil_scoped_acquire lock; - auto res = ::py::object((*guardCb)(**guardArgs, ***guardKwargs)).cast(); - // Release references immediately while we hold the GIL, instead of waiting - // for the lambda destructor to relock the GIL. - *guardKwargs = {}; - *guardArgs = {}; - *guardCb = {}; - return res; + GILAcquire acquire; + return invokeCatchPythonError( + sharedCb.takeInner(), + *sharedArgs.takeInner(), + **sharedKwArgs.takeInner()).cast(); }; // If there is a strand attached to that callable, we use it but we cannot use @@ -79,7 +76,7 @@ void exportAsync(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; m.def("runAsync", &async, "callback"_a, @@ -102,8 +99,11 @@ void exportAsync(::py::module& m) "Set the callback used by the periodic task, this function can only be " "called once.\n" ":param callable: a python callable, could be a method or a function.")) - .def("setUsPeriod", &PeriodicTask::setUsPeriod, - call_guard(), "usPeriod"_a, + .def("setUsPeriod", + [](PeriodicTask& task, qi::int64_t usPeriod) { + task.setPeriod(qi::MicroSeconds(usPeriod)); + }, + call_guard(), "usPeriod"_a, doc("Set the call interval in microseconds.\n" "This call will wait until next callback invocation to apply the " "change.\n" @@ -112,10 +112,10 @@ void exportAsync(::py::module& m) ".. code-block:: python\n" "\n" " task.stop()\n" - " task.setUsPeriod()\n" + " task.setUsPeriod(100)\n" " task.start()\n" ":param usPeriod: the period in microseconds")) - .def("start", &PeriodicTask::start, call_guard(), + .def("start", &PeriodicTask::start, call_guard(), "immediate"_a, doc("Start the periodic task at specified period. No effect if " "already running.\n" @@ -124,7 +124,7 @@ void exportAsync(::py::module& m) ".. warning::\n" " concurrent calls to start() and stop() will result in " "undefined behavior.")) - .def("stop", &PeriodicTask::stop, call_guard(), + .def("stop", &PeriodicTask::stop, call_guard(), doc("Stop the periodic task. When this function returns, the callback " "will not be called " "anymore. Can be called from within the callback function\n" @@ -132,12 +132,12 @@ void exportAsync(::py::module& m) " concurrent calls to start() and stop() will result in " "undefined behavior.")) .def("asyncStop", &PeriodicTask::asyncStop, - call_guard(), + call_guard(), doc("Request for periodic task to stop asynchronously.\n" "Can be called from within the callback function.")) .def( "compensateCallbackTime", &PeriodicTask::compensateCallbackTime, - call_guard(), "compensate"_a, + call_guard(), "compensate"_a, doc( ":param compensate: boolean. True to activate the compensation.\n" "When compensation is activated, call interval will take into account " @@ -146,7 +146,7 @@ void exportAsync(::py::module& m) " when the callback is longer than the specified period, " "compensation will result in the callback being called successively " "without pause.")) - .def("setName", &PeriodicTask::setName, call_guard(), + .def("setName", &PeriodicTask::setName, call_guard(), "name"_a, doc("Set name for debugging and tracking purpose")) .def("isRunning", &PeriodicTask::isRunning, doc(":returns: true if task is running")) diff --git a/src/pyclock.cpp b/src/pyclock.cpp index 027a5f7..bb5a514 100644 --- a/src/pyclock.cpp +++ b/src/pyclock.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -31,7 +32,7 @@ void exportClock(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; m.def("clockNow", &now, doc(":returns: current timestamp on qi::Clock, as a number of nanoseconds")); diff --git a/src/pyexport.cpp b/src/pyexport.cpp index 1aa0aa5..3926141 100644 --- a/src/pyexport.cpp +++ b/src/pyexport.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -31,7 +32,7 @@ void exportAll(pybind11::module& module) { registerTypes(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; exportFuture(module); exportSignal(module); diff --git a/src/pyfuture.cpp b/src/pyfuture.cpp index 4c4e9af..dc4c6fb 100644 --- a/src/pyfuture.cpp +++ b/src/pyfuture.cpp @@ -52,47 +52,48 @@ void castIfNotVoid(const ::py::object&) {} template std::function(Args...)> toContinuation(const ::py::function& cb) { - ::py::gil_scoped_acquire lock; - GILGuardedObject guardedCb(cb); - auto callGuardedCb = [=](Args... args) mutable { - ::py::gil_scoped_acquire lock; + GILAcquire lock; + SharedObject sharedCb(cb); + auto callSharedCb = [=](Args... args) mutable { + GILAcquire lock; const auto handleExcept = ka::handle_exception_rethrow( exceptionLogVerbose( logCategory, "An exception occurred while executing a future continuation"), ka::type_t<::py::object>{}); const auto pyRes = - ka::invoke_catch(handleExcept, *guardedCb, std::forward(args)...); - // Release references immediately while we hold the GIL, instead of waiting - // for the lambda destructor to relock the GIL. - *guardedCb = {}; + ka::invoke_catch(handleExcept, [&] { + return invokeCatchPythonError( + sharedCb.takeInner(), + std::forward(args)...); + }); return castIfNotVoid(pyRes); }; auto strand = strandOfFunction(cb); if (strand) - return strand->schedulerFor(std::move(callGuardedCb)); - return futurizeOutput(std::move(callGuardedCb)); + return strand->schedulerFor(std::move(callSharedCb)); + return futurizeOutput(std::move(callSharedCb)); } void addCallback(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; fut.connect(std::move(cont)); } Future then(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; return fut.then(std::move(cont)).unwrap(); } Future andThen(Future fut, const ::py::function& cb) { auto cont = toContinuation(cb); - ::py::gil_scoped_release _unlock; + GILRelease _unlock; return fut.andThen(std::move(cont)).unwrap(); } @@ -134,7 +135,7 @@ void exportFuture(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; enum_(m, "FutureState") .value("None", FutureState_None) @@ -162,30 +163,30 @@ void exportFuture(::py::module& m) } return prom; }), - call_guard(), + call_guard(), "on_cancel"_a = none(), doc(":param on_cancel: a function that will be called when a cancel is requested on the future.\n")) .def("setCanceled", &Promise::setCanceled, - call_guard(), + call_guard(), doc("Set the state of the promise to Canceled.")) .def("setError", &Promise::setError, - call_guard(), + call_guard(), "error"_a, doc("Set the error of the promise.")) .def("setValue", &Promise::setValue, - call_guard(), + call_guard(), "value"_a, doc("Set the value of the promise.")) .def("future", &Promise::future, - call_guard(), + call_guard(), doc("Get a future tied to the promise. You can get multiple futures from the same promise.")) .def("isCancelRequested", &Promise::isCancelRequested, - call_guard(), + call_guard(), doc(":returns: True if the future associated with the promise asked for cancellation.")); class_(m, "Future") @@ -194,7 +195,7 @@ void exportFuture(::py::module& m) .def("value", static_cast(&Future::value), - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Block until the future is ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" @@ -202,7 +203,7 @@ void exportFuture(::py::module& m) ":raises: a RuntimeError if the timeout is reached or the future has error.")) .def("error", &Future::error, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Block until the future is ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" @@ -212,44 +213,44 @@ void exportFuture(::py::module& m) .def("wait", static_cast(&Future::wait), - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc("Wait for the future to be ready.\n\n" ":param timeout: a time in milliseconds. Optional.\n" ":returns: a :data:`qi.FutureState`.")) .def("hasError", &Future::hasError, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc(":param timeout: a time in milliseconds. Optional.\n" ":returns: true iff the future has an error.\n" ":raise: a RuntimeError if the timeout is reached.")) .def("hasValue", &Future::hasValue, - call_guard(), + call_guard(), "timeout"_a = FutureTimeout_Infinite, doc(":param timeout: a time in milliseconds. Optional.\n" ":returns: true iff the future has a value.\n" ":raise: a RuntimeError if the timeout is reached.")) .def("cancel", &Future::cancel, - call_guard(), + call_guard(), doc("Ask for cancellation.")) .def("isFinished", &Future::isFinished, - call_guard(), + call_guard(), doc("Return true if the future is not running anymore (i.e. if hasError or hasValue or isCanceled).")) .def("isRunning", &Future::isRunning, - call_guard(), + call_guard(), doc("Return true if the future is still running.")) .def("isCanceled", &Future::isCanceled, - call_guard(), + call_guard(), doc("Return true if the future is canceled.")) .def("isCancelable", [](const Future&) { return true; }, - call_guard(), + call_guard(), doc(":returns: always true, all future are cancellable now\n" ".. deprecated:: 1.5.0\n")) @@ -279,12 +280,12 @@ void exportFuture(::py::module& m) ":returns: a future that will contain the return value of the callback.")) .def("unwrap", &qi::py::unwrap, - call_guard(), + call_guard(), doc("If this is a Future of a Future of X, return a Future of X.\n\n" "The state of both futures is forwarded and cancel requests are forwarded to the appropriate future.")); m.def("futureBarrier", &qi::py::futureBarrier, - call_guard(), + call_guard(), doc("Return a future that will be set with all the futures given as argument when they are\n" " all finished. This is useful to wait for a bunch of Futures at once.\n\n" ":param futureList: A list of Futures to wait for\n" diff --git a/src/pylog.cpp b/src/pylog.cpp index 4da952b..5e935f6 100644 --- a/src/pylog.cpp +++ b/src/pylog.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,7 @@ void exportLog(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; enum_(m, "LogLevel") .value("Silent", LogLevel_Silent) @@ -39,12 +40,12 @@ void exportLog(::py::module& m) log::log(level, name.c_str(), message.c_str(), file.c_str(), func.c_str(), line); }, - call_guard(), "level"_a, "name"_a, "message"_a, + call_guard(), "level"_a, "name"_a, "message"_a, "file"_a, "func"_a, "line"_a); m.def("setFilters", [](const std::string& filters) { log::addFilters(filters); }, - call_guard(), "filters"_a, + call_guard(), "filters"_a, doc("Set log filtering options.\n" "Each rule can be:\n\n" " +CAT: enable category CAT\n\n" @@ -60,7 +61,7 @@ void exportLog(::py::module& m) ":param filters: List of rules separated by colon.")); m.def("setContext", [](int context) { qi::log::setContext(context); }, - call_guard(), "context"_a, + call_guard(), "context"_a, doc(" 1 : Verbosity \n" " 2 : ShortVerbosity \n" " 4 : Date \n" @@ -77,7 +78,7 @@ void exportLog(::py::module& m) ":param context: A bitfield (sum of described values).")); m.def("setLevel", [](LogLevel level) { log::setLogLevel(level); }, - call_guard(), "level"_a, + call_guard(), "level"_a, doc( "Sets the threshold for the logger to level. " "Logging messages which are less severe than level will be ignored. " diff --git a/src/pymodule.cpp b/src/pymodule.cpp index 4cf04cd..5465cc0 100644 --- a/src/pymodule.cpp +++ b/src/pymodule.cpp @@ -24,7 +24,7 @@ namespace ::py::object call(::py::object obj, ::py::str name, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return obj.attr(name)(*args, **kwargs); } @@ -42,7 +42,7 @@ ::py::object getModule(const std::string& name) ::py::list listModules() { - const auto modules = invokeGuarded<::py::gil_scoped_release>(&qi::listModules); + const auto modules = invokeGuarded(&qi::listModules); return castToPyObject(AnyReference::from(modules)); } @@ -52,7 +52,7 @@ void exportObjectFactory(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; m.def("module", &getModule, doc(":returns: an object that represents the requested module.\n")); diff --git a/src/pyobject.cpp b/src/pyobject.cpp index 42076ef..9bc1019 100644 --- a/src/pyobject.cpp +++ b/src/pyobject.cpp @@ -41,7 +41,7 @@ constexpr static const auto overloadArgName = "_overload"; ::py::object call(const Object& obj, std::string funcName, ::py::args args, ::py::kwargs kwargs) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (auto optOverload = extractKeywordArg(kwargs, overloadArgName)) funcName = *optOverload; @@ -54,7 +54,7 @@ ::py::object call(const Object& obj, std::string funcName, Promise prom; { - ::py::gil_scoped_release _unlock; + GILRelease _unlock; auto metaCallFut = obj.metaCall(funcName, argsValue.asTupleValuePtr(), async ? MetaCallType_Queued : MetaCallType_Direct); @@ -77,7 +77,7 @@ std::string docString(const MetaMethod& method) void populateMethods(::py::object pyobj, const Object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& metaObj = obj.metaObject(); const auto& methodMap = metaObj.methodMap(); @@ -121,7 +121,7 @@ void populateSignals(::py::object pyobj, const Object& obj) continue; ::py::setattr(pyobj, signalName.c_str(), - castToPyObject(new detail::ProxySignal{ obj, signal.uid() })); + castToPyObject(detail::ProxySignal{ obj, signal.uid() })); } } @@ -138,7 +138,7 @@ void populateProperties(::py::object pyobj, const Object& obj) continue; ::py::setattr(pyobj, propName.c_str(), - castToPyObject(new detail::ProxyProperty{ obj, prop.uid() })); + castToPyObject(detail::ProxyProperty{ obj, prop.uid() })); } } @@ -149,7 +149,7 @@ using GenericFunction = std::function<::py::object(::py::args)>; // @pre `cargs.size() > 0`, arguments must at least contain a `DynamicObject` // on which the function is to be called. AnyReference callPythonMethod(const AnyReferenceVector& cargs, - const GILGuardedObject& method) + const SharedObject<::py::function>& method) { auto it = cargs.begin(); const auto cargsEnd = cargs.end(); @@ -158,7 +158,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, QI_ASSERT_TRUE(it != cargsEnd); ++it; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::tuple args(std::distance(it, cargsEnd)); ::py::size_t i = 0; @@ -170,7 +170,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, // Convert Python future object into a C++ Future, to allow libqi to unwrap // it. - const ::py::object ret = (*method)(*args); + const ::py::object ret = invokeCatchPythonError(method.inner(), *args); if (::py::isinstance(ret)) return AnyValue::from(ret.cast()).release(); return AnyReference::from(ret).content().clone(); @@ -185,7 +185,7 @@ AnyReference callPythonMethod(const AnyReferenceVector& cargs, // the number of positional parameters the function accepts. std::string methodDefaultParametersSignature(const ::py::function& method) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Returns a Signature object (see // https://docs.python.org/3/library/inspect.html#inspect.Signature). @@ -226,7 +226,7 @@ boost::optional registerMethod(DynamicObjectBuilder& gob, const ::py::function& method, std::string parametersSignature) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (boost::starts_with(name, "__")) { @@ -259,7 +259,7 @@ boost::optional registerMethod(DynamicObjectBuilder& gob, return gob.xAdvertiseMethod(mmb, AnyFunction::fromDynamicFunction( boost::bind(callPythonMethod, _1, - GILGuardedObject(method)))); + SharedObject(method)))); } } // namespace @@ -269,16 +269,17 @@ namespace detail boost::optional readObjectUid(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; - const ::py::bytes qiObjectUid = ::py::getattr(obj, qiObjectUidAttributeName, ::py::none()); - if (!qiObjectUid.is_none()) - return deserializeObjectUid(static_cast(qiObjectUid)); - return {}; + GILAcquire lock; + const auto qiObjectUidObj = ::py::getattr(obj, qiObjectUidAttributeName, ::py::none()); + if (qiObjectUidObj.is_none()) + return {}; + const auto qiObjectUid = qiObjectUidObj.cast(); + return deserializeObjectUid(qiObjectUid); } void writeObjectUid(const pybind11::object& obj, const ObjectUid& uid) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const ::py::bytes uidData = serializeObjectUid(uid); ::py::setattr(obj, qiObjectUidAttributeName, uidData); } @@ -309,7 +310,7 @@ ::py::object toPyObject(Object obj) Object toObject(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Check the few known Object types that a Python object can be. if (::py::isinstance(obj)) @@ -332,7 +333,7 @@ Object toObject(const ::py::object& obj) : ObjectThreadingModel_SingleThread); const auto attrKeys = ::py::reinterpret_steal<::py::list>(PyObject_Dir(obj.ptr())); - for (const ::py::handle& pyAttrKey : attrKeys) + for (const auto& pyAttrKey : attrKeys) { QI_ASSERT_TRUE(pyAttrKey); QI_ASSERT_FALSE(pyAttrKey.is_none()); @@ -341,7 +342,7 @@ Object toObject(const ::py::object& obj) const auto attrKey = pyAttrKey.cast(); auto memberName = attrKey; - const ::py::object attr = obj.attr(pyAttrKey); + const auto attr = obj.attr(pyAttrKey); if (attr.is_none()) { qiLogVerbose() << "The object attribute '" << attrKey @@ -392,11 +393,7 @@ Object toObject(const ::py::object& obj) // This is a useless callback, needed to keep a reference on the python object. // When the GenericObject is destroyed, the reference is released. - GILGuardedObject guardedObj(obj); - Object anyobj = gob.object([guardedObj](GenericObject*) mutable { - ::py::gil_scoped_acquire lock; - *guardedObj = {}; - }); + Object anyobj = gob.object([sharedObj = SharedObject(obj)](GenericObject*) {}); // If there was no ObjectUid stored in the python object, store a new one. if (!maybeObjectUid) @@ -417,17 +414,17 @@ void exportObject(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Object", dynamic_attr()) - .def(self == self, call_guard()) - .def(self != self, call_guard()) - .def(self < self, call_guard()) - .def(self <= self, call_guard()) - .def(self > self, call_guard()) - .def(self >= self, call_guard()) - .def("__bool__", &Object::isValid, call_guard()) - .def("isValid", &Object::isValid, call_guard()) + .def(self == self, call_guard()) + .def(self != self, call_guard()) + .def(self < self, call_guard()) + .def(self <= self, call_guard()) + .def(self > self, call_guard()) + .def(self >= self, call_guard()) + .def("__bool__", &Object::isValid, call_guard()) + .def("isValid", &Object::isValid, call_guard()) .def("call", [](Object& obj, const std::string& funcName, ::py::args args, ::py::kwargs kwargs) { @@ -443,7 +440,7 @@ void exportObject(::py::module& m) "funcName"_a) .def("metaObject", [](const Object& obj) { return AnyReference::from(obj.metaObject()); }, - call_guard()); + call_guard()); // TODO: .def("post") // TODO: .def("setProperty") // TODO: .def("property") diff --git a/src/pypath.cpp b/src/pypath.cpp index 85965a1..8d02ec6 100644 --- a/src/pypath.cpp +++ b/src/pypath.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,14 +24,14 @@ void exportPath(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; - m.def("sdkPrefix", &path::sdkPrefix, call_guard(), + m.def("sdkPrefix", &path::sdkPrefix, call_guard(), doc(":returns: The SDK prefix path. It is always a complete, native " "path.\n")); m.def( - "findBin", &path::findBin, call_guard(), "name"_a, + "findBin", &path::findBin, call_guard(), "name"_a, "searchInPath"_a = false, doc("Look for a binary in the system.\n" ":param name: string. The full name of the binary, or just the name.\n" @@ -40,14 +41,14 @@ void exportPath(::py::module& m) "otherwise.")); m.def( - "findLib", &path::findLib, call_guard(), "name"_a, + "findLib", &path::findLib, call_guard(), "name"_a, doc("Look for a library in the system.\n" ":param name: string. The full name of the library, or just the name.\n" ":returns: the complete, native path to the file found. An empty string " "otherwise.")); m.def( - "findConf", &path::findConf, call_guard(), + "findConf", &path::findConf, call_guard(), "application"_a, "file"_a, "excludeUserWritablePath"_a = false, doc("Look for a configuration file in the system.\n" ":param application: string. The name of the application.\n" @@ -59,7 +60,7 @@ void exportPath(::py::module& m) "otherwise.")); m.def( - "findData", &path::findData, call_guard(), + "findData", &path::findData, call_guard(), "application"_a, "file"_a, "excludeUserWritablePath"_a = false, doc("Look for a file in all dataPaths(application) directories. Return the " "first match.\n" @@ -76,7 +77,7 @@ void exportPath(::py::module& m) [](const std::string& applicationName, const std::string& pattern) { return path::listData(applicationName, pattern); }, - call_guard(), "applicationName"_a, "pattern"_a, + call_guard(), "applicationName"_a, "pattern"_a, doc("List data files matching the given pattern in all " "dataPaths(application) directories.\n" " For each match, return the occurrence from the first dataPaths prefix." @@ -91,12 +92,12 @@ void exportPath(::py::module& m) m.def("listData", [](const std::string& applicationName) { return path::listData(applicationName); }, - call_guard(), "applicationName"_a); + call_guard(), "applicationName"_a); m.def("confPaths", [](const std::string& applicationName) { return path::confPaths(applicationName); }, - call_guard(), + call_guard(), "applicationName"_a, doc("Get the list of directories used when searching for " "configuration files for the given application.\n" @@ -108,12 +109,12 @@ void exportPath(::py::module& m) " nor that they are writable.")); m.def("confPaths", [] { return path::confPaths(); }, - call_guard()); + call_guard()); m.def("dataPaths", [](const std::string& applicationName) { return path::dataPaths(applicationName); }, - call_guard(), + call_guard(), "applicationName"_a, doc("Get the list of directories used when searching for " "configuration files for the given application.\n" @@ -125,10 +126,10 @@ void exportPath(::py::module& m) " nor that they are writable.")); m.def("dataPaths", [] { return path::dataPaths(); }, - call_guard()); + call_guard()); m.def("binPaths", [] { return path::binPaths(); }, - call_guard(), + call_guard(), doc( ":returns: The list of directories used when searching for binaries.\n" ".. warning::\n" @@ -137,44 +138,44 @@ void exportPath(::py::module& m) m.def( "libPaths", [] { return path::libPaths(); }, - call_guard(), + call_guard(), doc(":returns: The list of directories used when searching for libraries.\n\n" ".. warning::\n" " You should not assume those directories exist," " nor that they are writable.")); m.def("setWritablePath", &path::detail::setWritablePath, - call_guard(), "path"_a, + call_guard(), "path"_a, doc("Set the writable files path for users.\n" ":param path: string. A path on the system.\n" "Use an empty path to reset it to its default value.")); m.def("userWritableDataPath", &path::userWritableDataPath, - call_guard(), "applicationName"_a, "fileName"_a, + call_guard(), "applicationName"_a, "fileName"_a, doc("Get the writable data files path for users.\n" ":param applicationName: string. Name of the application.\n" ":param fileName: string. Name of the file.\n" ":returns: The file path.")); m.def("userWritableConfPath", &path::userWritableConfPath, - call_guard(), "applicationName"_a, "fileName"_a, + call_guard(), "applicationName"_a, "fileName"_a, doc("Get the writable configuration files path for users.\n" ":param applicationName: string. Name of the application.\n" ":param fileName: string. Name of the file.\n" ":returns: The file path.")); m.def("sdkPrefixes", [] { return path::detail::getSdkPrefixes(); }, - call_guard(), + call_guard(), doc("List of SDK prefixes.\n" ":returns: The list of sdk prefixes.")); m.def("addOptionalSdkPrefix", &path::detail::addOptionalSdkPrefix, - call_guard(), "prefix"_a, + call_guard(), "prefix"_a, doc("Add a new SDK path.\n" ":param: an sdk prefix.")); m.def("clearOptionalSdkPrefix", &path::detail::clearOptionalSdkPrefix, - call_guard(), + call_guard(), doc("Clear all optional sdk prefixes.")); } diff --git a/src/pyproperty.cpp b/src/pyproperty.cpp index 9e39a45..ad91c24 100644 --- a/src/pyproperty.cpp +++ b/src/pyproperty.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -27,7 +28,7 @@ pybind11::object propertyConnect(Property& prop, const pybind11::function& pyCallback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::signalConnect(prop, pyCallback, async); } @@ -36,7 +37,7 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::proxySignalConnect(prop.object, prop.propertyId, callback, async); } @@ -45,13 +46,13 @@ ::py::object proxyPropertyConnect(detail::ProxyProperty& prop, detail::ProxyProperty::~ProxyProperty() { // The destructor can lock when waiting for callbacks to end. - ::py::gil_scoped_release unlock; + GILRelease unlock; object.reset(); } bool isProperty(const pybind11::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return ::py::isinstance(obj) || ::py::isinstance(obj); } @@ -61,36 +62,36 @@ void exportProperty(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; using PropertyPtr = std::unique_ptr; class_(m, "Property") .def(init([] { return new Property(TypeInterface::fromSignature("m")); }), - call_guard()) + call_guard()) .def(init([](const std::string& sig) { return PropertyPtr(new Property(TypeInterface::fromSignature(sig)), DeleteOutsideGIL()); }), - "signature"_a, call_guard()) + "signature"_a, call_guard()) .def("value", [](const Property& prop, bool async) { const auto fut = prop.value().async(); - gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - arg(asyncArgName) = false, call_guard(), + arg(asyncArgName) = false, call_guard(), doc("Return the value stored inside the property.")) .def("setValue", [](Property& prop, AnyValue value, bool async) { const auto fut = toFuture(prop.setValue(std::move(value)).async()); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - call_guard(), "value"_a, arg(asyncArgName) = false, + call_guard(), "value"_a, arg(asyncArgName) = false, doc("Set the value of the property.")) .def("addCallback", &propertyConnect, "cb"_a, arg(asyncArgName) = false, @@ -107,7 +108,7 @@ void exportProperty(::py::module& m) [](Property& prop, SignalLink id, bool async) { return detail::signalDisconnect(prop, id, async); }, - call_guard(), "id"_a, arg(asyncArgName) = false, + call_guard(), "id"_a, arg(asyncArgName) = false, doc("Disconnect the callback associated to id.\n\n" ":param id: the connection id returned by :method:connect or " ":method:addCallback\n" @@ -117,7 +118,7 @@ void exportProperty(::py::module& m) [](Property& prop, bool async) { return detail::signalDisconnectAll(prop, async); }, - call_guard(), arg(asyncArgName) = false, + call_guard(), arg(asyncArgName) = false, doc("Disconnect all subscribers associated to the property.\n\n" "This function should be used with caution, as it may also remove " "subscribers that were added by other callers.\n\n" @@ -127,18 +128,18 @@ void exportProperty(::py::module& m) .def("value", [](const detail::ProxyProperty& prop, bool async) { const auto fut = prop.object.property(prop.propertyId).async(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, - call_guard(), arg(asyncArgName) = false) + call_guard(), arg(asyncArgName) = false) .def("setValue", [](detail::ProxyProperty& prop, object pyValue, bool async) { AnyValue value(unwrapAsRef(pyValue)); - gil_scoped_release unlock; + GILRelease unlock; const auto fut = toFuture(prop.object.setProperty(prop.propertyId, std::move(value)) .async()); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); }, "value"_a, arg(asyncArgName) = false) diff --git a/src/pysession.cpp b/src/pysession.cpp index 97648f4..e54c1de 100644 --- a/src/pysession.cpp +++ b/src/pysession.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -35,7 +36,7 @@ ::py::object makeSession() { constexpr const auto category = "qi.python.session.deleter"; constexpr const auto msg = "Waiting for the shared pointer destruction..."; - ::py::gil_scoped_release x; + GILRelease x; auto fut = std::async(std::launch::async, [=](std::unique_ptr) { while (!wptr.expired()) { @@ -52,7 +53,7 @@ ::py::object makeSession() auto ptr = SessionPtr(new qi::Session{}, Deleter{}); boost::get_deleter(ptr)->wptr = ka::weak_ptr(ptr); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return py::makeSession(ptr); } @@ -60,7 +61,7 @@ ::py::object makeSession() ::py::object makeSession(SessionPtr sess) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return toPyObject(sess); } @@ -68,7 +69,7 @@ void exportSession(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; m.def("Session", []{ return makeSession(); }); } diff --git a/src/pysignal.cpp b/src/pysignal.cpp index 5ae7897..b6c3e9d 100644 --- a/src/pysignal.cpp +++ b/src/pysignal.cpp @@ -5,11 +5,10 @@ #include #include +#include #include #include #include -#include -#include #include #include @@ -27,15 +26,15 @@ namespace constexpr static const auto asyncArgName = "_async"; -AnyReference dynamicCallFunction(const GILGuardedObject& func, +AnyReference dynamicCallFunction(const SharedObject<::py::function>& func, const AnyReferenceVector& args) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::list pyArgs(args.size()); ::py::size_t i = 0; for (const auto& arg : args) pyArgs[i++] = castToPyObject(arg); - (*func)(*pyArgs); + invokeCatchPythonError(func.inner(), *pyArgs); return AnyValue::makeVoid().release(); } @@ -45,12 +44,12 @@ ::py::object connect(F&& connect, const ::py::function& pyCallback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto strand = strandOfFunction(pyCallback); SignalSubscriber subscriber(AnyFunction::fromDynamicFunction( boost::bind(dynamicCallFunction, - GILGuardedObject(pyCallback), + SharedObject(pyCallback), _1)), strand.get()); subscriber.setCallType(MetaCallType_Auto); @@ -65,7 +64,7 @@ ::py::object proxySignalConnect(detail::ProxySignal& sig, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return detail::proxySignalConnect(sig.object, sig.signalId, callback, async); } @@ -77,7 +76,7 @@ namespace detail detail::ProxySignal::~ProxySignal() { // The destructor can lock waiting for callbacks to end. - ::py::gil_scoped_release unlock; + GILRelease unlock; object.reset(); } @@ -86,10 +85,10 @@ ::py::object signalConnect(SignalBase& sig, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return connect( [&](const SignalSubscriber& sub) { - ::py::gil_scoped_release unlock; + GILRelease unlock; return sig.connectAsync(sub).andThen(FutureCallbackType_Sync, [](const SignalSubscriber& sub) { return sub.link(); @@ -105,7 +104,7 @@ ::py::object signalDisconnect(SignalBase& sig, SignalLink id, bool async) return AnyValue::from(success); }); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -116,7 +115,7 @@ ::py::object signalDisconnectAll(SignalBase& sig, bool async) return AnyValue::from(success); }); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -125,10 +124,10 @@ ::py::object proxySignalConnect(const AnyObject& obj, const ::py::function& callback, bool async) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return connect( [&](const SignalSubscriber& sub) { - ::py::gil_scoped_release unlock; + GILRelease unlock; return obj.connect(signalId, sub).async(); }, callback, async); @@ -139,10 +138,10 @@ ::py::object proxySignalDisconnect(const AnyObject& obj, bool async) { const auto fut = [&] { - ::py::gil_scoped_release unlock; + GILRelease unlock; return toFuture(obj.disconnect(id)); }(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; return resultObject(fut, async); } @@ -150,7 +149,7 @@ ::py::object proxySignalDisconnect(const AnyObject& obj, bool isSignal(const ::py::object& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return ::py::isinstance(obj) || ::py::isinstance(obj); } @@ -160,7 +159,7 @@ void exportSignal(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; using SignalPtr = std::unique_ptr; class_(m, "Signal") @@ -171,12 +170,12 @@ void exportSignal(::py::module& m) onSub = qi::futurizeOutput(onConnect); return new Signal(signature, onSub); }), - call_guard(), + call_guard(), "signature"_a = "m", "onConnect"_a = std::function()) .def( "connect", &detail::signalConnect, - call_guard(), + call_guard(), "callback"_a, arg(asyncArgName) = false, doc( "Connect the signal to a callback, the callback will be called each " @@ -187,13 +186,13 @@ void exportSignal(::py::module& m) ":returns: the connection id of the registered callback.")) .def("disconnect", &detail::signalDisconnect, - call_guard(), "id"_a, arg(asyncArgName) = false, + call_guard(), "id"_a, arg(asyncArgName) = false, doc("Disconnect the callback associated to id.\n" ":param id: the connection id returned by connect.\n" ":returns: true on success.")) .def("disconnectAll", &detail::signalDisconnectAll, - call_guard(), arg(asyncArgName) = false, + call_guard(), arg(asyncArgName) = false, doc("Disconnect all subscribers associated to the property.\n\n" "This function should be used with caution, as it may also remove " "subscribers that were added by other callers.\n\n" @@ -203,7 +202,7 @@ void exportSignal(::py::module& m) [](Signal& sig, args pyArgs) { const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); - gil_scoped_release unlock; + GILRelease unlock; sig.trigger(args); }, doc("Trigger the signal")); @@ -220,7 +219,7 @@ void exportSignal(::py::module& m) [](detail::ProxySignal& sig, args pyArgs) { const auto args = AnyReference::from(pyArgs).content().asTupleValuePtr(); - gil_scoped_release unlock; + GILRelease unlock; sig.object.metaPost(sig.signalId, args); }); } diff --git a/src/pystrand.cpp b/src/pystrand.cpp index fb0ad89..0e1b2d8 100644 --- a/src/pystrand.cpp +++ b/src/pystrand.cpp @@ -5,6 +5,7 @@ #include #include +#include #include qiLogCategory("qi.python.strand"); @@ -32,17 +33,17 @@ bool isUnboundMethod(const ::py::function& func, const ::py::object& object) QI_ASSERT_TRUE(func); QI_ASSERT_TRUE(object); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto dir = ::py::reinterpret_steal<::py::list>(PyObject_Dir(object.ptr())); - for (const ::py::handle attrName : dir) + for (const auto& attrName : dir) { - const ::py::handle attr = object.attr(attrName); + const auto attr = object.attr(attrName); QI_ASSERT_TRUE(attr); if (!PyMethod_Check(attr.ptr())) continue; - const ::py::handle unboundMethod = PyMethod_GET_FUNCTION(attr.ptr()); + const auto unboundMethod = ::py::reinterpret_borrow<::py::object>(PyMethod_GET_FUNCTION(attr.ptr())); if (func.is(unboundMethod)) return true; } @@ -77,7 +78,7 @@ ::py::object getPartialSelf(const ::py::function& partialFunc) { QI_ASSERT_TRUE(partialFunc); - ::py::gil_scoped_acquire lock; + GILAcquire lock; constexpr const char* argsAttr = "args"; constexpr const char* funcAttr = "func"; @@ -147,7 +148,7 @@ ::py::object getSelf(const ::py::function& func) { QI_ASSERT_TRUE(func); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto self = getMethodSelf(func); if (!self.is_none()) return self; @@ -158,7 +159,7 @@ ::py::object getSelf(const ::py::function& func) StrandPtr strandOfFunction(const ::py::function& func) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return strandOf(getSelf(func)); } @@ -166,7 +167,7 @@ StrandPtr strandOf(const ::py::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (obj.is_none()) return {}; @@ -202,7 +203,7 @@ bool isMultithreaded(const ::py::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyqisig = ::py::getattr(obj, objectAttributeThreadingName, ::py::none()); if (pyqisig.is_none()) return false; @@ -213,7 +214,7 @@ void exportStrand(::py::module& m) { using namespace ::py; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Strand") .def(init([] { diff --git a/src/pytranslator.cpp b/src/pytranslator.cpp index 7843d35..0eeec57 100644 --- a/src/pytranslator.cpp +++ b/src/pytranslator.cpp @@ -5,6 +5,7 @@ #include #include +#include #include namespace py = pybind11; @@ -19,21 +20,21 @@ void exportTranslator(::py::module& m) using namespace ::py; using namespace ::py::literals; - gil_scoped_acquire lock; + GILAcquire lock; class_(m, "Translator") .def(init([](const std::string& name) { return new Translator(name); }), - call_guard(), "name"_a) - .def("translate", &Translator::translate, call_guard(), + call_guard(), "name"_a) + .def("translate", &Translator::translate, call_guard(), "msg"_a, "domain"_a = "", "locale"_a = "", "context"_a = "", doc("Translate a message from a domain to a locale.")) .def("translate", &Translator::translateContext, - call_guard(), "msg"_a, "context"_a, + call_guard(), "msg"_a, "context"_a, doc("Translate a message with a context.")) .def("setCurrentLocale", &Translator::setCurrentLocale, - call_guard(), "locale"_a, doc("Set the locale.")) + call_guard(), "locale"_a, doc("Set the locale.")) .def("setDefaultDomain", &Translator::setDefaultDomain, "domain"_a, - call_guard(), doc("Set the domain.")) + call_guard(), doc("Set the domain.")) .def("addDomain", &Translator::addDomain, "domain"_a, doc("Add a new domain.")); } diff --git a/src/pytypes.cpp b/src/pytypes.cpp index fe41bfa..41795d3 100644 --- a/src/pytypes.cpp +++ b/src/pytypes.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -27,16 +28,6 @@ namespace py namespace { -struct ObjectDecRef -{ - ::py::handle obj; - void operator()() const - { - ::py::gil_scoped_acquire lock; - obj.dec_ref(); - } -}; - template boost::optional<::py::object> tryToCastObjectTo(ObjectTypeInterface* type, void* ptr) @@ -56,20 +47,20 @@ struct ValueToPyObject void visitUnknown(AnyReference value) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // Encapsulate the value in Capsule. result = ::py::capsule(value.rawValue()); } void visitVoid() { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::none(); } void visitInt(int64_t value, bool isSigned, int byteSize) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; // byteSize is 0 when the value is a boolean. if (byteSize == 0) result = ::py::bool_(static_cast(value)); @@ -80,13 +71,13 @@ struct ValueToPyObject void visitFloat(double value, int /*byteSize*/) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::float_(value); } void visitString(char* data, size_t len) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (!data) { @@ -121,7 +112,7 @@ struct ValueToPyObject void visitList(AnyIterator it, AnyIterator end) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::list l; for (; it != end; ++it) l.append(unwrapValue(*it)); @@ -135,7 +126,7 @@ struct ValueToPyObject void visitMap(AnyIterator it, AnyIterator end) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict d; for (; it != end; ++it) d[unwrapValue((*it)[0])] = (*it)[1]; @@ -150,7 +141,7 @@ struct ValueToPyObject const auto type = go.type; const auto ptr = type->ptrFromStorage(&go.value); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (auto obj = tryToCastObjectTo(type, ptr)) { result = *obj; @@ -167,7 +158,7 @@ struct ValueToPyObject void visitAnyObject(AnyObject& obj) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = py::toPyObject(obj); } @@ -182,7 +173,7 @@ struct ValueToPyObject { const auto len = tuple.size(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (annotations.empty()) { // Unnamed tuple @@ -203,7 +194,7 @@ struct ValueToPyObject void visitDynamic(AnyReference pointee) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = unwrapValue(pointee); } @@ -212,7 +203,7 @@ struct ValueToPyObject /* TODO: zerocopy, sub-buffers... */ const auto dataWithSize = value.asRaw(); - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = ::py::reinterpret_steal<::py::object>( PyByteArray_FromStringAndSize(dataWithSize.first, dataWithSize.second)); } @@ -224,7 +215,7 @@ struct ValueToPyObject void visitOptional(AnyReference v) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; result = unwrapValue(v.content()); } @@ -324,7 +315,7 @@ AnyReference associateValueToObj(::py::object& obj, T value) ::py::object unwrapValue(AnyReference val) { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::object result; ValueToPyObject tpo(result); typeDispatch(tpo, val); @@ -355,20 +346,20 @@ class ObjectInterfaceBase : public Interface if (ptr) return ptr; - ::py::gil_scoped_acquire lock; + GILAcquire lock; return new Storage; } void* clone(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; return new Storage(asObject(&storage)); } void destroy(void* storage) override { destroyDisownedReferences(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; delete asObjectPtr(&storage); } @@ -379,7 +370,7 @@ class ObjectInterfaceBase : public Interface bool less(void* a, void* b) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& objA = asObject(&a); const auto& objB = asObject(&b); return objA < objB; @@ -391,14 +382,14 @@ class DynamicInterface : public ObjectInterfaceBaseasObjectPtr(&storage); return unwrapAsRef(*obj); } void set(void** storage, AnyReference src) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = unwrapValue(src); } }; @@ -410,7 +401,7 @@ class IntInterface : public ObjectInterfaceBase std::int64_t get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); return numericConvertBound(::py::cast(obj)); } @@ -418,7 +409,7 @@ class IntInterface : public ObjectInterfaceBase void set(void** storage, std::int64_t val) override { QI_ASSERT_NOT_NULL(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::int_(static_cast(val)); } @@ -434,7 +425,7 @@ class FloatInterface : public ObjectInterfaceBaseasObject(&storage); return numericConvertBound(::py::cast(obj)); } @@ -442,7 +433,7 @@ class FloatInterface : public ObjectInterfaceBaseasObject(storage) = ::py::float_(static_cast(val)); } @@ -455,7 +446,7 @@ class BoolInterface : public ObjectInterfaceBase public: int64_t get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); return static_cast(::py::cast(obj)); } @@ -463,7 +454,7 @@ class BoolInterface : public ObjectInterfaceBase void set(void** storage, int64_t val) override { QI_ASSERT_NOT_NULL(storage); - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::bool_(static_cast(val)); } @@ -477,14 +468,14 @@ class StrInterface : public ObjectInterfaceBaseasObject(&storage); return makeManagedString(std::string(obj)); } void set(void** storage, const char* ptr, size_t sz) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; this->asObject(storage) = ::py::str(ptr, sz); } }; @@ -496,7 +487,7 @@ class StringBufferInterface : public ObjectInterfaceBaseasObject(&storage); const auto info = obj.request(); QI_ASSERT_TRUE(info.ndim == 1); @@ -506,7 +497,7 @@ class StringBufferInterface : public ObjectInterfaceBaseasObject(storage) = ::py::bytes(ptr, sz); } }; @@ -527,15 +518,17 @@ class StructuredIterableInterface std::vector get(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); std::vector res; res.reserve(_size); for (const ::py::handle itemHandle : obj) { - auto item = ::py::reinterpret_borrow<::py::object>(itemHandle); - res.push_back(AnyValue::from(item).release().rawValue()); + const auto item = ::py::reinterpret_borrow<::py::object>(itemHandle); + const auto itemRef = AnyValue::from(item).release(); + storeDisownedReference(storage, itemRef); + res.push_back(itemRef.rawValue()); } return res; } @@ -544,14 +537,16 @@ class StructuredIterableInterface { QI_ASSERT_TRUE(index < _size); - ::py::gil_scoped_acquire lock; + GILAcquire lock; const auto& obj = this->asObject(&storage); // AppleClang 8 wrongly requires a ForwardIterator on `std::next`, which // `pybind11::iterator` is not. We use advance instead. auto it = obj.begin(); std::advance(it, index); - const auto item = *it; - return AnyValue::from(item).release().rawValue(); + const auto item = ::py::reinterpret_borrow<::py::object>(*it); + const auto itemRef = AnyValue::from(item).release(); + storeDisownedReference(storage, itemRef); + return itemRef.rawValue(); } void set(void** /*storage*/, const std::vector&) override @@ -584,7 +579,7 @@ class ListInterface : public ObjectInterfaceBase auto* listStorage = iter.first; const auto index = iter.second; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ListType list = listType->asObject(&listStorage); const ::py::object element = list[index]; @@ -600,20 +595,13 @@ class ListInterface : public ObjectInterfaceBase using DefaultImpl = DefaultTypeImplMethods>; - void* initializeStorage(void* ptr = nullptr) override - { - return DefaultImpl::initializeStorage(ptr); - } - + void* initializeStorage(void* ptr = nullptr) override { return DefaultImpl::initializeStorage(ptr); } void* clone(void* storage) override { return DefaultImpl::clone(storage); } - void destroy(void* storage) override - { - destroyDisownedReferences(storage); - return DefaultImpl::destroy(storage); - } + void destroy(void* storage) override { return DefaultImpl::destroy(storage); } const TypeInfo& info() override { return DefaultImpl::info(); } void* ptrFromStorage(void** s) override { return DefaultImpl::ptrFromStorage(s); } bool less(void* a, void* b) override { return DefaultImpl::less(a, b); } + Iterator* asIterPtr(void** storage) { return static_cast(ptrFromStorage(storage)); } Iterator& asIter(void** storage) { return *asIterPtr(storage); } }; @@ -625,14 +613,14 @@ class ListInterface : public ObjectInterfaceBase size_t size(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ListType list = this->asObject(&storage); return list.size(); } void pushBack(void** storage, void* valueStorage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::object obj = this->asObject(storage); if (::py::isinstance<::py::list>(obj)) { @@ -677,7 +665,7 @@ class DictInterface: public ObjectInterfaceBase auto* dictStorage = iter.first; const auto index = iter.second; - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = dictType->asObject(&dictStorage); // AppleClang 8 wrongly requires a ForwardIterator on `std::next`, which // `pybind11::iterator` is not. We use advance instead. @@ -685,12 +673,13 @@ class DictInterface: public ObjectInterfaceBase std::advance(it, index); const auto key = ::py::reinterpret_borrow<::py::object>(it->first); const auto element = ::py::reinterpret_borrow<::py::object>(it->second); - const auto keyElementPair = std::make_pair(key, element); - auto ref = AnyReference::from(keyElementPair).clone(); + auto keyRef = AnyReference::from(key); + auto elementRef = AnyReference::from(element); + auto pairRef = makeGenericTuple({keyRef, elementRef}); // Store the disowned reference with the list as a context instead of the // iterator because the reference might outlive the iterator. - storeDisownedReference(dictStorage, ref); - return ref; + storeDisownedReference(dictStorage, pairRef); + return pairRef; } void next(void** storage) override { ++asIter(storage).second; } @@ -698,21 +687,13 @@ class DictInterface: public ObjectInterfaceBase using DefaultImpl = DefaultTypeImplMethods>; - void* initializeStorage(void* ptr = nullptr) override - { - return DefaultImpl::initializeStorage(ptr); - } - + void* initializeStorage(void* ptr = nullptr) override { return DefaultImpl::initializeStorage(ptr); } void* clone(void* storage) override { return DefaultImpl::clone(storage); } - void destroy(void* storage) override - { - destroyDisownedReferences(storage); - return DefaultImpl::destroy(storage); - } - + void destroy(void* storage) override { return DefaultImpl::destroy(storage); } const TypeInfo& info() override { return DefaultImpl::info(); } void* ptrFromStorage(void** s) override { return DefaultImpl::ptrFromStorage(s); } bool less(void* a, void* b) override { return DefaultImpl::less(a, b); } + Iterator* asIterPtr(void** storage) { return static_cast(ptrFromStorage(storage)); } Iterator& asIter(void** storage) { return *asIterPtr(storage); } }; @@ -729,14 +710,14 @@ class DictInterface: public ObjectInterfaceBase size_t size(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(&storage); return dict.size(); } AnyIterator begin(void* storage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(&storage); return AnyValue(AnyReference(instance(), new Iterator(storage, 0)), // Do not copy, but free the value, so basically the AnyValue @@ -755,7 +736,7 @@ class DictInterface: public ObjectInterfaceBase void insert(void** storage, void* keyStorage, void* valueStorage) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(storage); ::py::object key = keyType()->asObject(&keyStorage); ::py::object value = elementType()->asObject(&valueStorage); @@ -764,7 +745,7 @@ class DictInterface: public ObjectInterfaceBase AnyReference element(void** storage, void* keyStorage, bool autoInsert) override { - ::py::gil_scoped_acquire lock; + GILAcquire lock; ::py::dict dict = this->asObject(storage); ::py::object key = keyType()->asObject(&keyStorage); @@ -792,7 +773,7 @@ AnyReference unwrapAsRef(pybind11::object& obj) { QI_ASSERT_TRUE(obj); - ::py::gil_scoped_acquire lock; + GILAcquire lock; if (obj.is_none()) // The "void" value in AnyValue has no storage, so we can just release it diff --git a/tests/common.hpp b/tests/common.hpp index 2773ca8..fb4f11c 100644 --- a/tests/common.hpp +++ b/tests/common.hpp @@ -9,6 +9,7 @@ #define QIPYTHON_TESTS_COMMON_HPP #include +#include #include #include #include @@ -44,25 +45,25 @@ struct Execute { Execute() { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; _locals.emplace(); } ~Execute() { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; _locals.reset(); } void exec(const std::string& str) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; pybind11::exec(str, pybind11::globals(), *_locals); } pybind11::object eval(const std::string& str) { - pybind11::gil_scoped_acquire lock; + GILAcquire lock; return pybind11::eval(str, pybind11::globals(), *_locals); } diff --git a/tests/moduletest.cpp b/tests/moduletest.cpp index 023ea24..717b773 100644 --- a/tests/moduletest.cpp +++ b/tests/moduletest.cpp @@ -20,6 +20,77 @@ int Mouse::squeak() return 18; } +class Purr +{ +public: + Purr(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " purr constructor "; + ++(*counter); + } + ~Purr() + { + qiLogInfo() << this << " purr destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " purring"; + } + qi::Property volume; +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Purr, run, volume); + +class Sleep +{ +public: + Sleep(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " sleep constructor "; + ++(*counter); + } + ~Sleep() + { + qiLogInfo() << this << " sleep destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " sleeping"; + } +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Sleep, run); + +class Play +{ +public: + Play(std::shared_ptr> counter) : counter(counter) + { + qiLogInfo() << this << " play constructor "; + ++(*counter); + } + ~Play() + { + qiLogInfo() << this << " play destructor "; + --(*counter); + } + void run() + { + qiLogInfo() << this << " playing"; + } + qi::Signal caught; +private: + std::shared_ptr> counter; +}; + +QI_REGISTER_OBJECT(Play, run, caught); + class Cat { public: @@ -35,24 +106,65 @@ class Cat return boost::make_shared(); } + boost::shared_ptr makePurr() const + { + return boost::make_shared(purrCounter); + } + + boost::shared_ptr makeSleep() const + { + return boost::make_shared(sleepCounter); + } + + boost::shared_ptr makePlay() const + { + return boost::make_shared(playCounter); + } + + void order(qi::AnyObject /*action*/) const + { + // Cats do not follow orders, they do nothing. + } + + int nbPurr() + { + return purrCounter->load(); + } + + int nbSleep() + { + return sleepCounter->load(); + } + + int nbPlay() + { + return playCounter->load(); + } + qi::Property hunger; qi::Property boredom; qi::Property cuteness; + + std::shared_ptr> purrCounter = std::make_shared>(0); + std::shared_ptr> sleepCounter = std::make_shared>(0); + std::shared_ptr> playCounter = std::make_shared>(0); }; -QI_REGISTER_OBJECT(Cat, meow, cloneMe, hunger, boredom, cuteness); +QI_REGISTER_OBJECT(Cat, meow, cloneMe, hunger, boredom, cuteness, + makePurr, makeSleep, makePlay, order, + nbPurr, nbSleep, nbPlay); Cat::Cat() { qiLogInfo() << "Cat constructor"; } -Cat::Cat(const std::string& s) +Cat::Cat(const std::string& s) : Cat() { qiLogInfo() << "Cat string constructor: " << s; } -Cat::Cat(const qi::SessionPtr& s) +Cat::Cat(const qi::SessionPtr& s) : Cat() { qiLogInfo() << "Cat string constructor with session"; s->services(); // SEGV? diff --git a/tests/test_guard.cpp b/tests/test_guard.cpp index 40b261a..d9d835c 100644 --- a/tests/test_guard.cpp +++ b/tests/test_guard.cpp @@ -39,66 +39,74 @@ TEST_F(InvokeGuardedTest, ExecutesFunctionWithGuard) EXPECT_EQ(52, r); } -using GuardedTest = GuardTest; - -TEST_F(GuardedTest, GuardsConstruction) +TEST(GILAcquire, IsReentrant) { - using T = GuardedTest; - - struct Check - { - int i; - Check(int i) : i(i) { EXPECT_TRUE(T::guarded); } - explicit operator bool() const { return true; } - }; - - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded g(42); - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, g->i); + qi::py::GILAcquire acq0; QI_IGNORE_UNUSED(acq0); + qi::py::GILRelease rel; QI_IGNORE_UNUSED(rel); + qi::py::GILAcquire acq1; QI_IGNORE_UNUSED(acq1); + qi::py::GILAcquire acq2; QI_IGNORE_UNUSED(acq2); + SUCCEED(); } -TEST_F(GuardedTest, GuardsCopyConstruction) +TEST(GILRelease, IsReentrant) { - using T = GuardedTest; + qi::py::GILRelease rel0; QI_IGNORE_UNUSED(rel0); + qi::py::GILAcquire acq; QI_IGNORE_UNUSED(acq); + qi::py::GILRelease rel1; QI_IGNORE_UNUSED(rel1); + qi::py::GILRelease rel2; QI_IGNORE_UNUSED(rel2); + SUCCEED(); +} - struct Check +struct SharedObject : testing::Test +{ + SharedObject() { - int i = 0; - Check(int i) : i(i) {} - Check(const Check& o) - : i(o.i) - { EXPECT_TRUE(T::guarded); } - explicit operator bool() const { return true; } - }; + // GIL is only required for creation of the inner object. + object = [&]{ + qi::py::GILAcquire lock; + return pybind11::capsule( + &this->destroyed, + [](void* destroyed){ + *reinterpret_cast(destroyed) = true; + } + ); + }(); + } + + ~SharedObject() + { + if (object) { + qi::py::GILAcquire lock; + object = {}; + } + } - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded a(42); - qi::py::detail::Guarded b(a); - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, a->i); - EXPECT_EQ(42, b->i); -} + bool destroyed = false; + pybind11::capsule object; +}; -TEST_F(GuardedTest, GuardsCopyAssignment) +TEST_F(SharedObject, KeepsRefCount) { - using T = GuardedTest; - - struct Check + std::optional sharedObject = qi::py::SharedObject(std::move(object)); + ASSERT_FALSE(object); // object has been released. + EXPECT_FALSE(destroyed); // inner object is still alive. { - int i = 0; - Check(int i) : i(i) {} - Check& operator=(const Check& o) - { EXPECT_TRUE(T::guarded); i = o.i; return *this; } - explicit operator bool() const { return true; } - }; + auto sharedObjectCpy = *sharedObject; // copy the shared object locally. + EXPECT_FALSE(destroyed); // inner object is maintained by both copies. + sharedObject.reset(); + EXPECT_FALSE(destroyed); // inner object is maintained by the copy. + } + EXPECT_TRUE(destroyed); // inner object has been destroyed. +} - EXPECT_FALSE(T::guarded); - qi::py::detail::Guarded a(42); - qi::py::detail::Guarded b(37); - EXPECT_FALSE(T::guarded); - b = a; - EXPECT_FALSE(T::guarded); - EXPECT_EQ(42, a->i); - EXPECT_EQ(42, b->i); +TEST_F(SharedObject, TakeInnerStealsInnerRefCount) +{ + std::optional sharedObject = qi::py::SharedObject(std::move(object)); + auto inner = sharedObject->takeInner(); + EXPECT_FALSE(sharedObject->inner()); // inner object is null. + sharedObject.reset(); + EXPECT_FALSE(destroyed); // inner object is still alive. + qi::py::GILAcquire lock; // release local inner object, which requires the GIL. + inner = {}; + EXPECT_TRUE(destroyed); // inner object has been destroyed. } diff --git a/tests/test_module.cpp b/tests/test_module.cpp index 0a11339..95b8b13 100644 --- a/tests/test_module.cpp +++ b/tests/test_module.cpp @@ -4,6 +4,7 @@ #include #include #include +#include qiLogCategory("TestQiPython.Module"); @@ -14,7 +15,7 @@ namespace void globalExec(const std::string& str) { - py::gil_scoped_acquire _lock; + qi::py::GILAcquire _lock; py::exec(str.c_str()); } @@ -42,7 +43,7 @@ MATCHER(ModuleEq, "") { return sameModule(std::get<0>(arg), std::get<1>(arg)); } TEST(Module, listModules) { - py::gil_scoped_acquire _lock; + qi::py::GILAcquire _lock; auto qi = py::globals()["qi"]; auto modulesFromPython = qi::AnyReference::from(qi.attr("listModules")()).to>(); diff --git a/tests/test_object.cpp b/tests/test_object.cpp index 761310a..b84dbcf 100644 --- a/tests/test_object.cpp +++ b/tests/test_object.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "common.hpp" namespace py = pybind11; @@ -39,7 +40,7 @@ struct ReadObjectUidTest : qi::py::test::Execute, testing::Test {}; TEST_F(ReadObjectUidTest, ReturnsEmptyIfAbsent) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto a = eval("object()"); auto objectUid = qi::py::detail::readObjectUid(a); EXPECT_FALSE(objectUid); @@ -47,7 +48,7 @@ TEST_F(ReadObjectUidTest, ReturnsEmptyIfAbsent) TEST_F(ReadObjectUidTest, ReturnsValueIfPresent) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(declareTypeAWithUid); auto a = eval("A()"); auto objectUid = qi::py::detail::readObjectUid(a); @@ -70,7 +71,7 @@ constexpr const std::array WriteObjectUidTest::data; TEST_F(WriteObjectUidTest, CanBeReadAfterWritten) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(R"( class A(object): pass @@ -85,7 +86,7 @@ class A(object): TEST_F(WriteObjectUidTest, CanBeReadAfterOverwritten) { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(declareTypeAWithUid); auto a = eval("A()"); @@ -144,7 +145,7 @@ class Cookies(object): template qi::AnyObject makeObject(Args&&... args) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto type = locals()["Cookies"]; return qi::py::toObject(type(std::forward(args)...)); } @@ -216,14 +217,14 @@ struct ObjectTest : testing::Test { object = boost::make_shared(); - py::gil_scoped_acquire lock; + GILAcquire lock; pyObject = qi::py::toPyObject(object); ASSERT_TRUE(pyObject); } void TearDown() override { - py::gil_scoped_acquire lock; + GILAcquire lock; pyObject.release().dec_ref(); } @@ -248,7 +249,7 @@ QI_REGISTER_OBJECT(ObjectTest::Muffins, count, bakedCount, baked) TEST_F(ObjectTest, IsValid) { - py::gil_scoped_acquire lock; + GILAcquire lock; { EXPECT_TRUE(py::bool_(pyObject)); EXPECT_TRUE(py::bool_(pyObject.attr("isValid")())); @@ -265,18 +266,18 @@ TEST_F(ObjectTest, IsValid) TEST_F(ObjectTest, Call) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto res = pyObject.attr("call")("count", 832).cast(); EXPECT_EQ("You have 832 muffins.", res); } TEST_F(ObjectTest, Async) { - py::gil_scoped_acquire lock; + GILAcquire lock; { const auto res = qi::py::test::toFutOf( py::cast(pyObject.attr("async")("count", 2356))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 2356 muffins.", res.value()); } @@ -284,7 +285,7 @@ TEST_F(ObjectTest, Async) { const auto res = qi::py::test::toFutOf(py::cast( pyObject.attr("call")("count", 32897, py::arg("_async") = true))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 32897 muffins.", res.value()); } @@ -294,7 +295,7 @@ using ToPyObjectTest = ObjectTest; TEST_F(ToPyObjectTest, MethodCanBeCalled) { - py::gil_scoped_acquire lock; + GILAcquire lock; { const auto res = pyObject.attr("count")(8).cast(); EXPECT_EQ("You have 8 muffins.", res); @@ -305,7 +306,7 @@ TEST_F(ToPyObjectTest, MethodCanBeCalled) const auto res = qi::py::test::toFutOf(py::cast( pyObject.attr("count")(5, py::arg("_async") = true))); - py::gil_scoped_release unlock; + GILRelease unlock; ASSERT_TRUE(test::finishesWithValue(res)); EXPECT_EQ("You have 5 muffins.", res.value()); } @@ -313,21 +314,21 @@ TEST_F(ToPyObjectTest, MethodCanBeCalled) TEST_F(ToPyObjectTest, PropertyIsExposed) { - py::gil_scoped_acquire lock; + GILAcquire lock; const py::object prop = pyObject.attr("bakedCount"); EXPECT_TRUE(qi::py::isProperty(prop)); } TEST_F(ToPyObjectTest, SignalIsExposed) { - py::gil_scoped_acquire lock; + GILAcquire lock; const py::object sig = pyObject.attr("baked"); EXPECT_TRUE(qi::py::isSignal(sig)); } TEST_F(ToPyObjectTest, FutureAsObjectIsReturnedAsPyFuture) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto res = pyObject.attr("count")(8).cast(); EXPECT_EQ("You have 8 muffins.", res); } diff --git a/tests/test_property.cpp b/tests/test_property.cpp index 5687550..03d3ee5 100644 --- a/tests/test_property.cpp +++ b/tests/test_property.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include "common.hpp" @@ -17,13 +18,13 @@ struct PropertyTest : qi::py::test::Execute, testing::Test { PropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type = py::globals()["qi"].attr("Property"); } ~PropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type.release().dec_ref(); } @@ -32,7 +33,7 @@ struct PropertyTest : qi::py::test::Execute, testing::Test TEST_F(PropertyTest, DefaultConstructedHasDynamicSignature) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyProp = type(); const auto prop = pyProp.cast(); const auto sigChildren = prop->signature().children(); @@ -42,7 +43,7 @@ TEST_F(PropertyTest, DefaultConstructedHasDynamicSignature) TEST_F(PropertyTest, ConstructedWithSignature) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto pyProp = type(qi::Signature::fromType(qi::Signature::Type_String).toString()); const auto prop = pyProp.cast(); @@ -55,14 +56,14 @@ struct ConstructedPropertyTest : PropertyTest { ConstructedPropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; pyProp = type("i"); prop = pyProp.cast(); } ~ConstructedPropertyTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; prop = nullptr; pyProp = {}; } @@ -78,7 +79,7 @@ TEST_F(PropertyValueTest, CanBeReadAfterSetFromCxx) auto setFut = prop->setValue(839).async(); ASSERT_TRUE(test::finishesWithValue(setFut)); - py::gil_scoped_acquire lock; + GILAcquire lock; const auto value = pyProp.attr("value")().cast(); EXPECT_EQ(839, value); } @@ -89,7 +90,7 @@ TEST_F(PropertyValueTest, ValueCanBeReadAfterSetFromCxxAsync) ASSERT_TRUE(test::finishesWithValue(setFut)); const auto valueFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp.attr("value")(py::arg("_async") = true).cast()); }(); @@ -103,7 +104,7 @@ using PropertySetValueTest = ConstructedPropertyTest; TEST_F(PropertySetValueTest, ValueCanBeReadFromCxx) { { - py::gil_scoped_acquire lock; + GILAcquire lock; pyProp.attr("setValue")(4893); } @@ -119,7 +120,7 @@ namespace TEST_F(PropertySetValueTest, ValueCanBeReadFromCxxAsync) { const auto setValueFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp.attr("setValue")(3423409, py::arg("_async") = true) .cast()); @@ -140,7 +141,7 @@ TEST_F(PropertyAddCallbackTest, CallbackIsCalledWhenValueIsSet) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(mockFn.AsStdFunction()).cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -159,7 +160,7 @@ TEST_F(PropertyAddCallbackTest, PythonCallbackIsCalledWhenValueIsSet) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; auto pyFn = eval("lambda f : lambda i : f(i * 2)"); const auto id = pyProp.attr("addCallback")(pyFn(mockFn.AsStdFunction())) .cast(); @@ -179,7 +180,7 @@ TEST_F(PropertyAddCallbackTest, CallbackIsCalledWhenValueIsSetAsync) StrictMock> mockFn; const auto idFut = [&] { - py::gil_scoped_acquire lock; + GILAcquire lock; return qi::py::test::toFutOf( pyProp .attr("addCallback")(mockFn.AsStdFunction(), py::arg("_async") = true) @@ -201,7 +202,7 @@ TEST_F(PropertyAddCallbackTest, CallbackThrowsWhenCalledDoesNotReportError) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(mockFn.AsStdFunction()).cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -214,7 +215,7 @@ TEST_F(PropertyAddCallbackTest, CallbackThrowsWhenCalledDoesNotReportError) ASSERT_TRUE(test::finishesWithValue(setValueFut)); ASSERT_TRUE(test::finishesWithValue(cp.fut())); - py::gil_scoped_acquire lock; + GILAcquire lock; EXPECT_EQ(nullptr, PyErr_Occurred()); } @@ -222,7 +223,7 @@ using PropertyDisconnectTest = ConstructedPropertyTest; TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnected) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(py::cpp_function([](int) {})) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -233,20 +234,20 @@ TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnected) TEST_F(PropertyDisconnectTest, AddedCallbackCanBeDisconnectedAsync) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pyProp.attr("addCallback")(py::cpp_function([](int) {})) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); auto success = qi::py::test::toFutOf( pyProp.attr("disconnect")(id, py::arg("_async") = true) - .cast()); + .cast()).value(); EXPECT_TRUE(success); } TEST_F(PropertyDisconnectTest, DisconnectRandomSignalLinkFails) { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = qi::SignalLink(42); auto success = pyProp.attr("disconnect")(id).cast(); diff --git a/tests/test_qipython.cpp b/tests/test_qipython.cpp index 157226d..1ac1495 100644 --- a/tests/test_qipython.cpp +++ b/tests/test_qipython.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,10 @@ qiLogCategory("TestQiPython"); namespace py = pybind11; +PYBIND11_EMBEDDED_MODULE(qi, m) { + qi::py::exportAll(m); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); @@ -23,13 +28,12 @@ int main(int argc, char **argv) boost::optional app; app.emplace(argc, argv); - py::module m("qi"); - qi::py::exportAll(m); - py::globals()["qi"] = m; + py::globals()["qi"] = py::module::import("qi"); + int ret = EXIT_FAILURE; { - py::gil_scoped_release unlock; + qi::py::GILRelease unlock; ret = RUN_ALL_TESTS(); // Destroy the application outside of the GIL to avoid deadlocks, but while diff --git a/tests/test_qipython_local_interpreter.cpp b/tests/test_qipython_local_interpreter.cpp new file mode 100644 index 0000000..581255a --- /dev/null +++ b/tests/test_qipython_local_interpreter.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_EMBEDDED_MODULE(test_local_interpreter, m) { + struct ObjectDtorOutsideGIL + { + ~ObjectDtorOutsideGIL() + { + qi::py::GILRelease _rel; + // nothing. + } + + }; + pybind11::class_(m, "ObjectDtorOutsideGIL") + .def(pybind11::init([]{ + return std::make_unique(); + })); +} + +TEST(InterpreterFinalize, GarbageObjectDtorOutsideGIL) +{ + pybind11::scoped_interpreter interp; + pybind11::globals()["qitli"] = pybind11::module::import("test_local_interpreter"); + pybind11::exec("obj = qitli.ObjectDtorOutsideGIL()"); +} + +// This test checks that concurrent uses of GIL guards during finalization +// of the interpreter does not cause crashes. +// +// It uses 2 threads: a main thread, and a second native thread. +// +// The main thread starts an interpreter, starts the second thread and +// releases the GIL. The second thread acquires the GIL, then releases +// it, and waits for the interpreter to finalize. Once it is finalized, +// it tries to reacquire the GIL, and then release it, according to +// GIL guards destructors. +// +// Horizontal lines are synchronization points between the threads. +// +// main thread (T1) +// ~~~~~~~~~~~~~~~~ +// ▼ +// ╔═══════╪════════╗ +// ║ interpreter ║ +// ----------------------------------------------> start native thread T2 +// ║ ╎ ║ native thread (T2) +// ║ ╎ ║ ~~~~~~~~~~~~~~~~~~ +// ║ ╎ ║ ▼ +// ║ ╔═════╪══════╗ ║ ╎ +// ║ ║ GILRelease ║ ║ ╎ -> T1 releases the GIL +// ----------------------------------------------> GIL shift T1 -> T2 +// ║ ║ ╎ ║ ║ ╔═══════╪════════╗ +// ║ ║ ╎ ║ ║ ║ GILAcquire ║ -> T2 acquires the GIL +// ║ ║ ╎ ║ ║ ║ ╔═════╪══════╗ ║ +// ║ ║ ╎ ║ ║ ║ ║ GILRelease ║ ║ -> T2 releases the GIL +// ----------------------------------------------> GIL shift T2 -> T1 +// ║ ╚═════╪══════╝ ║ ║ ║ ╎ ║ ║ -> T1 acquires the GIL +// ╚═══════╪════════╝ ║ ║ ╎ ║ ║ -> interpreter starts finalizing +// ----------------------------------------------> interpreter finalized +// ╎ ║ ╚═════╪══════╝ ║ -> T2 tries to reacquire GIL but fails, it's a noop. +// ╎ ╚═══════╪════════╝ -> T2 tries to release GIL, but it's a noop. +// ╎ ╎ +TEST(InterpreterFinalize, GILReleasedInOtherThread) +{ + // Synchronization mechanism for the first GIL shift, otherwise T1 will release and reacquire the + // GIL instantly without waiting for T2. + qi::Promise shift1Prom; + auto shift1Fut = shift1Prom.future(); + // Second GIL shift does not require an additional synchronization + // mechanism. Waiting for interpreter finalization ensures that the + // GIL was acquired back by thread T1. + // Synchronization mechanism for interpreter finalization. + qi::Promise finalizedProm; + std::future asyncFut; + { + pybind11::scoped_interpreter interp; + asyncFut = std::async( + std::launch::async, + [finalizedFut = finalizedProm.future(), shift1Prom]() mutable { + qi::py::GILAcquire acquire; + // First GIL shift is done. + shift1Prom.setValue(nullptr); + qi::py::GILRelease release; + finalizedFut.value(); + }); + qi::py::GILRelease release; + shift1Fut.value(); + } + finalizedProm.setValue(nullptr); + // Join the thread, wait for the task to finish and ensure no exception was thrown. + asyncFut.get(); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + qi::Application app(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/test_signal.cpp b/tests/test_signal.cpp index 001eaec..1bec90c 100644 --- a/tests/test_signal.cpp +++ b/tests/test_signal.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -18,7 +19,7 @@ struct DefaultConstructedSignalTest : qi::py::test::Execute, testing::Test { DefaultConstructedSignalTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; type = py::globals()["qi"].attr("Signal"); pySig = type(); sig = pySig.cast(); @@ -26,7 +27,7 @@ struct DefaultConstructedSignalTest : qi::py::test::Execute, testing::Test ~DefaultConstructedSignalTest() { - py::gil_scoped_acquire lock; + GILAcquire lock; sig = nullptr; pySig = {}; type = {}; @@ -43,7 +44,7 @@ TEST_F(DefaultConstructedSignalTest, AcceptsCallbackWithoutArgument) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pySig.attr("connect")(mockFn.AsStdFunction()) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -61,7 +62,7 @@ TEST_F(DefaultConstructedSignalTest, AcceptsCallbackWithAnyArgument) StrictMock> mockFn; { - py::gil_scoped_acquire lock; + GILAcquire lock; const auto id = pySig.attr("connect")(mockFn.AsStdFunction()) .cast(); EXPECT_NE(qi::SignalBase::invalidSignalLink, id); @@ -90,7 +91,7 @@ struct ConstructedThroughServiceSignalTest : qi::py::test::Execute, testing::Tes test::finishesWithValue(clientSession->connect(servSession->url()))); { - py::gil_scoped_acquire lock; + GILAcquire lock; exec(R"py( class Cookies(object): def __init__(self): @@ -113,7 +114,7 @@ class Cookies(object): void TearDown() override { - py::gil_scoped_acquire lock; + GILAcquire lock; pySig = {}; } @@ -140,7 +141,7 @@ TEST_F(ConstructedThroughServiceSignalTest, })))); { - py::gil_scoped_acquire lock; + GILAcquire lock; pySig(); } diff --git a/tests/test_types.cpp b/tests/test_types.cpp index 155e947..a3712e1 100644 --- a/tests/test_types.cpp +++ b/tests/test_types.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "common.hpp" @@ -25,14 +26,14 @@ struct PybindObjectCast { qi::AnyValue operator()(py::object obj) { - py::gil_scoped_acquire lock; + qi::py::GILAcquire lock; return obj.cast(); } }; template -struct ToAnyValueConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueConversionTest : qi::py::GILAcquire, testing::Test { template qi::AnyValue toAnyValue(T&& t) @@ -110,7 +111,7 @@ TYPED_TEST(ToAnyValueConversionTest, PyBytes) EXPECT_TRUE(v.template to().equal(py::bytes("donuts"))); } -struct ToAnyValueObjectConversionTest : py::gil_scoped_acquire, testing::Test {}; +struct ToAnyValueObjectConversionTest : qi::py::GILAcquire, testing::Test {}; TEST_F(ToAnyValueObjectConversionTest, AnyValueFromReturnsDynamic) { @@ -123,7 +124,7 @@ TEST_F(ToAnyValueObjectConversionTest, AnyValueFromReturnsDynamic) // to `PybindObjectCast` conversion with the underlying object which should // already be tested. -struct ToAnyValueListConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueListConversionTest : qi::py::GILAcquire, testing::Test { static std::vector toVec(qi::AnyReference listRef) { @@ -157,7 +158,7 @@ TEST_F(ToAnyValueListConversionTest, PybindObjectCastReturnsList) using DictTypes = testing::Types; template -struct ToAnyValueDictConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueDictConversionTest : qi::py::GILAcquire, testing::Test { static std::map toMap(qi::AnyReference map) { @@ -198,7 +199,7 @@ TYPED_TEST(ToAnyValueDictConversionTest, PybindObjectCastReturnsMap) using TupleTypes = testing::Types; template -struct ToAnyValueTupleConversionTest : py::gil_scoped_acquire, testing::Test +struct ToAnyValueTupleConversionTest : qi::py::GILAcquire, testing::Test { static std::tuple toTuple(qi::AnyReference tuple) { @@ -241,7 +242,7 @@ struct TypePassing : qi::py::test::Execute, session = qi::makeSession(); session->listenStandalone("tcp://127.0.0.1:0"); - py::gil_scoped_acquire lock; + qi::py::GILAcquire lock; locals()["sd"] = qi::py::makeSession(session); } @@ -386,20 +387,40 @@ TEST_F(TypePassing, Recursive) } } +TEST_F(TypePassing, ReverseList) +{ + exec( + "class TestService:\n" + " def func(self, list):\n" + // Test the iterator interface. + " for value in list:\n" + " assert(isinstance(value, str))\n" + " assert(list == ['hello', 'world'])\n" + ); + registerService(); + const std::vector list {"hello", "world"}; + getService().call("func", list); +} + + TEST_F(TypePassing, ReverseDict) { exec( "class TestService:\n" " def func(self, dict):\n" - " return dict == {'one' : 1, 'two' : 2, 'three' : 3}\n" + // Test the iterator interface. + " for key, value in dict.items():\n" + " assert(isinstance(key, str))\n" + " assert(isinstance(value, int))\n" + " assert(dict == {'one' : 1, 'two' : 2, 'three' : 3})\n" ); registerService(); - const std::map expected { + const std::map dict { {"one", 1}, {"two", 2}, {"three", 3}, }; - EXPECT_TRUE(getService().call("func", expected)); + getService().call("func", dict); } TEST_F(TypePassing, LogLevel)