diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 688d65c04..b5a0080a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,13 +6,13 @@ jobs: build-test: name: Build and Test runs-on: ${{ matrix.os }}-latest - timeout-minutes: 5 + timeout-minutes: 7 strategy: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: [3.6, 3.7, 3.8, 3.9] + python: ["3.6", "3.7", "3.8", "3.9"] platform: [x64] shutdown_mode: [Normal, Soft] @@ -44,19 +44,34 @@ jobs: - name: Build and Install run: | - python setup.py configure pip install -v . - - name: Python Tests - run: pytest + - name: Set Python DLL path (non Windows) + if: ${{ matrix.os != 'windows' }} + run: | + python -m pythonnet.find_libpython --export >> $GITHUB_ENV + + - name: Set Python DLL path (Windows) + if: ${{ matrix.os == 'windows' }} + run: | + python -m pythonnet.find_libpython --export | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/ - if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython + + - name: Python Tests (Mono) + if: ${{ matrix.os != 'windows' }} + run: pytest --runtime mono + + - name: Python Tests (.NET Core) + run: pytest --runtime netcore + + - name: Python Tests (.NET Framework) + if: ${{ matrix.os == 'windows' }} + run: pytest --runtime netfx - name: Python tests run from .NET run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ - if: ${{ matrix.os == 'windows' }} # Not working for others right now # TODO: Run perf tests # TODO: Run mono tests on Windows? diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml new file mode 100644 index 000000000..fcd8ca06e --- /dev/null +++ b/.github/workflows/nuget-preview.yml @@ -0,0 +1,60 @@ +name: NuGet Preview Release + +on: + schedule: + - cron: "5 4 3 */1 *" # once a month, at 4:05 on 3rd + workflow_dispatch: + +jobs: + release: + name: Release Preview + runs-on: ubuntu-latest + environment: NuGet + timeout-minutes: 10 + + env: + PYTHONNET_SHUTDOWN_MODE: Normal + + steps: + - name: Get Date + run: | + echo "DATE_VER=$(date "+%Y-%m-%d")" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + + - name: Build and Install + run: | + pip install -v . + + - name: Python Tests + run: pytest + env: + PYTHONNET_PYDLL: libpython3.8.so + + - name: Embedding tests + run: dotnet test --runtime any-ubuntu src/embed_tests/ + env: + PYTHONNET_PYDLL: libpython3.8.so + + - name: Pack + run: dotnet pack --configuration Release --version-suffix preview${{env.DATE_VER}} --output "Release-Preview" + + - name: Publish NuGet + run: dotnet nuget push --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_MONTHLY }} Release-Preview/*.nupkg + + # TODO: Run perf tests + # TODO: Run mono tests on Windows? diff --git a/.gitignore b/.gitignore index 673681317..cdb152157 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ /src/runtime/interopNative.cs -# Configuration data -configured.props - # General binaries and Build results *.dll *.exe diff --git a/AUTHORS.md b/AUTHORS.md index 167fd496c..912831836 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -55,6 +55,7 @@ - Meinrad Recheis ([@henon](https://github.com/henon)) - Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) +- Peter Kese ([@pkese](https://github.com/pkese)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) @@ -66,10 +67,10 @@ - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) -- William Sardar ([@williamsardar])(https://github.com/williamsardar) +- William Sardar ([@williamsardar](https://github.com/williamsardar)) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) -- ([@amos402]https://github.com/amos402) +- ([@amos402](https://github.com/amos402)) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) @@ -82,3 +83,4 @@ - ([@testrunner123](https://github.com/testrunner123)) - ([@DanBarzilian](https://github.com/DanBarzilian)) - ([@alxnull](https://github.com/alxnull)) +- ([@gpetrou](https://github.com/gpetrou)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a804e8c..da8f94774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). +- Add GetPythonThreadID and Interrupt methods in PythonEngine +- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) +- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec` +- Improved exception handling: + - exceptions can now be converted with codecs + - `InnerException` and `__cause__` are propagated properly ### Changed - Drop support for Python 2, 3.4, and 3.5 @@ -28,28 +34,52 @@ details about the cause of the failure to the regular method return value (unless they are passed with `ref` or `out` keyword). - BREAKING: Drop support for the long-deprecated CLR.* prefix. - `PyObject` now implements `IEnumerable` in addition to `IEnumerable` +- floating point values passed from Python are no longer silently truncated +when .NET expects an integer [#1342][i1342] +- More specific error messages for method argument mismatch +- BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. +- BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name +or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. +- BREAKING: `PyObject.Length()` now raises a `PythonException` when object does not support a concept of length. +- BREAKING: disabled implicit conversion from C# enums to Python `int` and back. +One must now either use enum members (e.g. `MyEnum.Option`), or use enum constructor +(e.g. `MyEnum(42)` or `MyEnum(42, True)` when `MyEnum` does not have a member with value 42). +- Sign Runtime DLL with a strong name +- Implement loading through `clr_loader` instead of the included `ClrModule`, enables + support for .NET Core +- .NET and Python exceptions are preserved when crossing Python/.NET boundary +- BREAKING: custom encoders are no longer called for instances of `System.Type` +- `PythonException.Restore` no longer clears `PythonException` instance. +- Replaced the old `__import__` hook hack with a PEP302-style Meta Path Loader ### Fixed -- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash -- Fix incorrect dereference in params array handling -- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) -- Fix `object[]` parameters taking precedence when should not in overload resolution -- Fixed a bug where all .NET class instances were considered Iterable -- Fix incorrect choice of method to invoke when using keyword arguments. -- Fix non-delegate types incorrectly appearing as callable. -- Indexers can now be used with interface objects -- Fixed a bug where indexers could not be used if they were inherited -- Made it possible to use `__len__` also on `ICollection<>` interface objects -- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions -- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects -- Fixed objects returned by enumerating `PyObject` being disposed too soon -- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling +- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Fixed a bug where all .NET class instances were considered Iterable +- Fix incorrect choice of method to invoke when using keyword arguments. +- Fix non-delegate types incorrectly appearing as callable. +- Indexers can now be used with interface objects +- Fixed a bug where indexers could not be used if they were inherited +- Made it possible to use `__len__` also on `ICollection<>` interface objects +- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions +- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects +- Fixed objects returned by enumerating `PyObject` being disposed too soon +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException - `import` may now raise errors with more detail than "No module named X" +- Exception stacktraces on `PythonException.StackTrace` are now properly formatted +- Providing an invalid type parameter to a generic type or method produces a helpful Python error +- Empty parameter names (as can be generated from F#) do not cause crashes +- Unicode strings with surrogates were truncated when converting from Python ### Removed - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) +- messages in `PythonException` no longer start with exception type +- support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 +(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) ## [2.5.0][] - 2020-06-14 @@ -809,3 +839,4 @@ This version improves performance on benchmarks significantly compared to 2.3. [i755]: https://github.com/pythonnet/pythonnet/pull/755 [p534]: https://github.com/pythonnet/pythonnet/pull/534 [i449]: https://github.com/pythonnet/pythonnet/issues/449 +[i1342]: https://github.com/pythonnet/pythonnet/issues/1342 diff --git a/Directory.Build.props b/Directory.Build.props index 5ad0c0e77..e0cd93ede 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,14 +4,18 @@ Copyright (c) 2006-2020 The Contributors of the Python.NET Project pythonnet Python.NET - 7.3 + 9.0 + false - + + all + runtime; build; native; contentfiles; analyzers + + all runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/LICENSE b/LICENSE index 19c31a12f..f3a638346 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2006-2020 the contributors of the Python.NET project +Copyright (c) 2006-2021 the contributors of the Python.NET project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), diff --git a/README.rst b/README.rst index a03468e7d..996bfab27 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,11 @@ pythonnet - Python.NET |gh shield| |appveyor shield| -|license shield| |pypi package version| |conda-forge version| |python supported shield| +|license shield| + +|pypi package version| |conda-forge version| |python supported shield| + +|nuget preview shield| |nuget release shield| Python.NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and @@ -41,6 +45,10 @@ module: Embedding Python in .NET ------------------------ +- You must set `Runtime.PythonDLL` property or `PYTHONNET_PYDLL` environment variable + starting with version 3.0, otherwise you will receive `TypeInitializationException`. + Typical values are `python38.dll` (Windows), `libpython3.8.dylib` (Mac), + `libpython3.8.so` (most other *nix). - All calls to python should be inside a ``using (Py.GIL()) {/* Your code here */}`` block. - Import python modules using ``dynamic mod = Py.Import("mod")``, then @@ -130,5 +138,9 @@ This project is supported by the `.NET Foundation :target: http://stackoverflow.com/questions/tagged/python.net .. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg :target: https://anaconda.org/conda-forge/pythonnet +.. |nuget preview shield| image:: https://img.shields.io/nuget/vpre/pythonnet + :target: https://www.nuget.org/packages/pythonnet/ +.. |nuget release shield| image:: https://img.shields.io/nuget/v/pythonnet + :target: https://www.nuget.org/packages/pythonnet/ .. |gh shield| image:: https://github.com/pythonnet/pythonnet/workflows/GitHub%20Actions/badge.svg :target: https://github.com/pythonnet/pythonnet/actions?query=branch%3Amaster diff --git a/appveyor.yml b/appveyor.yml index 1ad673ede..cc3815c62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,6 +32,7 @@ init: - set PY_VER=%PYTHON_VERSION:.=% - set PYTHON=C:\PYTHON%PY_VER% - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64) + - set PYTHONNET_PYDLL=%PYTHON%\python%PY_VER%.dll # Put desired Python version first in PATH - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% @@ -41,7 +42,6 @@ install: - pip install --upgrade -r requirements.txt --quiet build_script: - - python setup.py configure # Create clean `sdist`. Only used for releases - python setup.py --quiet sdist - python setup.py bdist_wheel diff --git a/clr.py b/clr.py index 711333dd2..20a975f96 100644 --- a/clr.py +++ b/clr.py @@ -2,40 +2,5 @@ Legacy Python.NET loader for backwards compatibility """ -def _get_netfx_path(): - import os, sys - - if sys.maxsize > 2 ** 32: - arch = "amd64" - else: - arch = "x86" - - return os.path.join(os.path.dirname(__file__), "pythonnet", "netfx", arch, "clr.pyd") - - -def _get_mono_path(): - import os, glob - - paths = glob.glob(os.path.join(os.path.dirname(__file__), "pythonnet", "mono", "clr.*so")) - return paths[0] - - -def _load_clr(): - import sys - from importlib import util - - if sys.platform == "win32": - path = _get_netfx_path() - else: - path = _get_mono_path() - - del sys.modules[__name__] - - spec = util.spec_from_file_location("clr", path) - clr = util.module_from_spec(spec) - spec.loader.exec_module(clr) - - sys.modules[__name__] = clr - - -_load_clr() +from pythonnet import load +load() diff --git a/pyproject.toml b/pyproject.toml index 83a58d126..b6df82f71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] requires = ["setuptools>=42", "wheel", "pycparser"] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +xfail_strict = true +testpaths = [ + "tests" +] diff --git a/pythonnet.sln b/pythonnet.sln index 4da4d7e99..eca470595 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -6,21 +6,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime", "src\runti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{819E089B-4770-400E-93C6-4F7A35F0EA12}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\Python.Test.csproj", "{14EF9518-5BB7-4F83-8686-015BD2CC788E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "tests\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig .gitignore = .gitignore CHANGELOG.md = CHANGELOG.md + LICENSE = LICENSE README.rst = README.rst EndProjectSection EndProject @@ -30,6 +29,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF- ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 .github\workflows\main.yml = .github\workflows\main.yml + .github\workflows\nuget-preview.yml = .github\workflows\nuget-preview.yml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" @@ -51,6 +51,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{BC426F42 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PythonTestsRunner", "src\python_tests_runner\Python.PythonTestsRunner.csproj", "{35CBBDEB-FC07-4D04-9D3E-F88FC180110B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{142A6752-C2C2-4F95-B982-193418001B65}" + ProjectSection(SolutionItems) = preProject + Directory.Build.props = Directory.Build.props + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -145,18 +150,6 @@ Global {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.ActiveCfg = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x86.Build.0 = Debug|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|Any CPU.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x64.Build.0 = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.ActiveCfg = Release|Any CPU - {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Release|x86.Build.0 = Release|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|Any CPU.Build.0 = Debug|Any CPU {35CBBDEB-FC07-4D04-9D3E-F88FC180110B}.Debug|x64.ActiveCfg = Debug|Any CPU diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 5c197e146..188980b8b 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -1,3 +1,60 @@ -def get_assembly_path(): - import os - return os.path.dirname(__file__) + "/runtime/Python.Runtime.dll" +import os +import sys +import clr_loader + +_RUNTIME = None +_LOADER_ASSEMBLY = None +_FFI = None +_LOADED = False + + +def set_runtime(runtime): + global _RUNTIME + if _LOADED: + raise RuntimeError("The runtime {runtime} has already been loaded".format(_RUNTIME)) + + _RUNTIME = runtime + + +def set_default_runtime() -> None: + if sys.platform == 'win32': + set_runtime(clr_loader.get_netfx()) + else: + set_runtime(clr_loader.get_mono()) + + +def load(): + global _FFI, _LOADED, _LOADER_ASSEMBLY + + if _LOADED: + return + + from os.path import join, dirname + + if _RUNTIME is None: + # TODO: Warn, in the future the runtime must be set explicitly, either + # as a config/env variable or via set_runtime + set_default_runtime() + + dll_path = join(dirname(__file__), "runtime", "Python.Runtime.dll") + + _LOADER_ASSEMBLY = _RUNTIME.get_assembly(dll_path) + + func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Initialize"] + if func(''.encode("utf8")) != 0: + raise RuntimeError("Failed to initialize Python.Runtime.dll") + + import atexit + atexit.register(unload) + + +def unload(): + global _RUNTIME + if _LOADER_ASSEMBLY is not None: + func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Shutdown"] + if func(b"") != 0: + raise RuntimeError("Failed to call Python.NET shutdown") + + if _RUNTIME is not None: + # TODO: Add explicit `close` to clr_loader + _RUNTIME = None diff --git a/pythonnet/find_libpython/__init__.py b/pythonnet/find_libpython/__init__.py new file mode 100644 index 000000000..3ae28970e --- /dev/null +++ b/pythonnet/find_libpython/__init__.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +# License +# +# Copyright 2018, Takafumi Arakaki +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, absolute_import + +from logging import getLogger +import ctypes.util +import functools +import os +import sys +import sysconfig + +logger = getLogger("find_libpython") + +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") +if SHLIB_SUFFIX is None: + if is_windows: + SHLIB_SUFFIX = ".dll" + else: + SHLIB_SUFFIX = ".so" +if is_apple: + # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. + # Let's not use the value from sysconfig. + SHLIB_SUFFIX = ".dylib" + + +def linked_libpython(): + """ + Find the linked libpython using dladdr (in *nix). + + Returns + ------- + path : str or None + A path to linked libpython. Return `None` if statically linked. + """ + if is_windows: + return _linked_libpython_windows() + return _linked_libpython_unix() + + +class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + +def _linked_libpython_unix(): + if not sysconfig.get_config_var("Py_ENABLE_SHARED"): + return None + + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + + dlinfo = Dl_info() + retcode = libdl.dladdr( + ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), + ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + path = os.path.realpath(dlinfo.dli_fname.decode()) + return path + + +def _linked_libpython_windows(): + """ + Based on: https://stackoverflow.com/a/16659821 + """ + from ctypes.wintypes import HANDLE, LPWSTR, DWORD + + GetModuleFileName = ctypes.windll.kernel32.GetModuleFileNameW + GetModuleFileName.argtypes = [HANDLE, LPWSTR, DWORD] + GetModuleFileName.restype = DWORD + + MAX_PATH = 260 + try: + buf = ctypes.create_unicode_buffer(MAX_PATH) + GetModuleFileName(ctypes.pythonapi._handle, buf, MAX_PATH) + return buf.value + except (ValueError, OSError): + return None + + + +def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows and name.startswith("lib"): + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def uniquifying(items): + """ + Yield items while excluding the duplicates and preserving the order. + + >>> list(uniquifying([1, 2, 1, 2, 3])) + [1, 2, 3] + """ + seen = set() + for x in items: + if x not in seen: + yield x + seen.add(x) + + +def uniquified(func): + """ Wrap iterator returned from `func` by `uniquifying`. """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return uniquifying(func(*args, **kwds)) + return wrapper + + +@uniquified +def candidate_names(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate file names of libpython. + + Yields + ------ + name : str + Candidate name libpython. + """ + LDLIBRARY = sysconfig.get_config_var("LDLIBRARY") + if LDLIBRARY and not LDLIBRARY.endswith(".a"): + yield LDLIBRARY + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY and not LIBRARY.endswith(".a"): + yield os.path.splitext(LIBRARY)[0] + suffix + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + # VERSION is X.Y in Linux/macOS and XY in Windows: + VERSION=(sysconfig.get_python_version() or + "{v.major}.{v.minor}".format(v=sys.version_info) or + sysconfig.get_config_var("VERSION")), + ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + + for stem in [ + "python{VERSION}{ABIFLAGS}".format(**sysdata), + "python{VERSION}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]: + yield dlprefix + stem + suffix + + + +@uniquified +def candidate_paths(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + + yield linked_libpython() + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var('LIBPL')) + append_truthy(lib_dirs, sysconfig.get_config_var('srcdir')) + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + # LIBPL seems to be the right config_var to use. It is the one + # used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.7.0/Misc/python-config.in#L55-L57 + # + # But we try other places just in case. + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + lib_basenames = list(candidate_names(suffix=suffix)) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + +# Possibly useful links: +# * https://packages.ubuntu.com/bionic/amd64/libpython3.6/filelist +# * https://github.com/Valloric/ycmd/issues/518 +# * https://github.com/Valloric/ycmd/pull/519 + + +def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + if is_apple: + return normalize_path(_remove_suffix_apple(path), + suffix=".so", is_apple=False) + return None + + +def _remove_suffix_apple(path): + """ + Strip off .so or .dylib. + + >>> _remove_suffix_apple("libpython.so") + 'libpython' + >>> _remove_suffix_apple("libpython.dylib") + 'libpython' + >>> _remove_suffix_apple("libpython3.7") + 'libpython3.7' + """ + if path.endswith(".dylib"): + return path[:-len(".dylib")] + if path.endswith(".so"): + return path[:-len(".so")] + return path + + +@uniquified +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. + + Yields + ------ + path : str + Existing path to a libpython. + """ + logger.debug("is_windows = %s", is_windows) + logger.debug("is_apple = %s", is_apple) + for path in candidate_paths(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + else: + logger.debug("Not found.") + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def print_all(items): + for x in items: + print(x) + + +def cli_find_libpython(cli_op, verbose, export): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig( + format="%(levelname)s %(message)s", + level=logging.DEBUG) + + if cli_op == "list-all": + print_all(finding_libpython()) + elif cli_op == "candidate-names": + print_all(candidate_names()) + elif cli_op == "candidate-paths": + print_all(p for p in candidate_paths() if p and os.path.isabs(p)) + else: + path = find_libpython() + if path is None: + return 1 + if export: + print(f"PYTHONNET_PYDLL={path}") + else: + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list-all", + action="store_const", dest="cli_op", const="list-all", + help="Print list of all paths found.") + group.add_argument( + "--candidate-names", + action="store_const", dest="cli_op", const="candidate-names", + help="Print list of candidate names of libpython.") + group.add_argument( + "--candidate-paths", + action="store_const", dest="cli_op", const="candidate-paths", + help="Print list of candidate paths of libpython.") + group.add_argument( + "--export", + action="store_true", + help="Print as an environment export expression" + ) + + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) diff --git a/pythonnet/find_libpython/__main__.py b/pythonnet/find_libpython/__main__.py new file mode 100644 index 000000000..031df43e1 --- /dev/null +++ b/pythonnet/find_libpython/__main__.py @@ -0,0 +1,2 @@ +from . import main +main() diff --git a/pythonnet/mono/.gitkeep b/pythonnet/mono/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/pythonnet/netfx/.gitkeep b/pythonnet/netfx/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/pythonnet/util/__init__.py b/pythonnet/util/__init__.py new file mode 100644 index 000000000..75d4bad8c --- /dev/null +++ b/pythonnet/util/__init__.py @@ -0,0 +1 @@ +from .find_libpython import find_libpython diff --git a/requirements.txt b/requirements.txt index 6f25858bc..f5aedfc3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ psutil coverage codecov -# Platform specific requirements wheel pycparser setuptools +clr-loader diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 19c6f9fc9..000000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[tool:pytest] -xfail_strict = True -# -r fsxX: show extra summary info for: (f)ailed, (s)kip, (x)failed, (X)passed -addopts = -r fsxX --color=yes --durations=5 diff --git a/setup.py b/setup.py index 06a26ef95..c74ca2c8c 100644 --- a/setup.py +++ b/setup.py @@ -8,79 +8,9 @@ import sys, os -BUILD_MONO = True -BUILD_NETFX = True - PY_MAJOR = sys.version_info[0] PY_MINOR = sys.version_info[1] -CONFIGURED_PROPS = "configured.props" - - -def _get_interop_filename(): - """interopXX.cs is auto-generated as part of the build. - For common windows platforms pre-generated files are included - as most windows users won't have Clang installed, which is - required to generate the file. - """ - interop_filename = "interop{0}{1}{2}.cs".format( - PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "") - ) - return os.path.join("src", "runtime", interop_filename) - - -# Write configuration to configured.props. This will go away once all of these -# can be decided at runtime. -def _write_configure_props(): - defines = [ - "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR), - ] - - if sys.platform == "win32": - defines.append("WINDOWS") - - if hasattr(sys, "abiflags"): - if "d" in sys.abiflags: - defines.append("PYTHON_WITH_PYDEBUG") - if "m" in sys.abiflags: - defines.append("PYTHON_WITH_PYMALLOC") - - # check the interop file exists, and create it if it doesn't - interop_file = _get_interop_filename() - if not os.path.exists(interop_file): - print("Creating {0}".format(interop_file)) - geninterop = os.path.join("tools", "geninterop", "geninterop.py") - check_call([sys.executable, geninterop, interop_file]) - - import xml.etree.ElementTree as ET - - proj = ET.Element("Project") - props = ET.SubElement(proj, "PropertyGroup") - f = ET.SubElement(props, "PythonInteropFile") - f.text = os.path.basename(interop_file) - - c = ET.SubElement(props, "ConfiguredConstants") - c.text = " ".join(defines) - - ET.ElementTree(proj).write(CONFIGURED_PROPS) - - -class configure(Command): - """Configure command""" - - description = "Configure the pythonnet build" - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - self.announce("Writing configured.props...", level=distutils.log.INFO) - _write_configure_props() - class DotnetLib: def __init__(self, name, path, **kwargs): @@ -121,7 +51,6 @@ def finalize_options(self): def run(self): dotnet_modules = self.distribution.dotnet_libs - self.run_command("configure") for lib in dotnet_modules: output = os.path.join( @@ -165,6 +94,7 @@ def run(self): # Add build_dotnet to the build tasks: from distutils.command.build import build as _build from setuptools.command.develop import develop as _develop +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools import Distribution import setuptools @@ -182,14 +112,22 @@ def install_for_development(self): return super().install_for_development() +class bdist_wheel(_bdist_wheel): + def finalize_options(self): + # Monkey patch bdist_wheel to think the package is pure even though we + # include DLLs + super().finalize_options() + self.root_is_pure = True + + # Monkey-patch Distribution s.t. it supports the dotnet_libs attribute Distribution.dotnet_libs = None cmdclass = { "build": build, "build_dotnet": build_dotnet, - "configure": configure, "develop": develop, + "bdist_wheel": bdist_wheel, } @@ -204,54 +142,6 @@ def install_for_development(self): ) ] -if BUILD_NETFX: - dotnet_libs.extend( - [ - DotnetLib( - "clrmodule-amd64", - "src/clrmodule/", - runtime="win-x64", - output="pythonnet/netfx/amd64", - rename={"clr.dll": "clr.pyd"}, - ), - DotnetLib( - "clrmodule-x86", - "src/clrmodule/", - runtime="win-x86", - output="pythonnet/netfx/x86", - rename={"clr.dll": "clr.pyd"}, - ), - ] - ) - -ext_modules = [] - -if BUILD_MONO: - try: - mono_libs = check_output( - "pkg-config --libs mono-2", shell=True, encoding="utf8" - ) - mono_cflags = check_output( - "pkg-config --cflags mono-2", shell=True, encoding="utf8" - ) - cflags = mono_cflags.strip() - libs = mono_libs.strip() - - # build the clr python module - clr_ext = Extension( - "pythonnet.mono.clr", - language="c++", - sources=["src/monoclr/clrmod.c"], - extra_compile_args=cflags.split(" "), - extra_link_args=libs.split(" "), - ) - ext_modules.append(clr_ext) - except Exception: - print( - "Failed to find mono libraries via pkg-config, skipping the Mono CLR loader" - ) - - setup( cmdclass=cmdclass, name="pythonnet", @@ -261,12 +151,10 @@ def install_for_development(self): license="MIT", author="The Contributors of the Python.NET Project", author_email="pythonnet@python.org", - packages=["pythonnet"], - install_requires=["pycparser"], + packages=["pythonnet", "pythonnet.find_libpython"], + install_requires=["clr_loader"], long_description=long_description, - # data_files=[("{install_platlib}", ["{build_lib}/pythonnet"])], py_modules=["clr"], - ext_modules=ext_modules, dotnet_libs=dotnet_libs, classifiers=[ "Development Status :: 5 - Production/Stable", @@ -277,6 +165,7 @@ def install_for_development(self): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs deleted file mode 100644 index 7b0387d46..000000000 --- a/src/clrmodule/ClrModule.cs +++ /dev/null @@ -1,113 +0,0 @@ -//============================================================================ -// This file replaces the hand-maintained stub that used to implement clr.dll. -// This is a line-by-line port from IL back to C#. -// We now use RGiesecke.DllExport on the required static init method so it can be -// loaded by a standard CPython interpreter as an extension module. When it -// is loaded, it bootstraps the managed runtime integration layer and defers -// to it to do initialization and put the clr module into sys.modules, etc. - -// The "USE_PYTHON_RUNTIME_*" defines control what extra evidence is used -// to help the CLR find the appropriate Python.Runtime assembly. - -// If defined, the "pythonRuntimeVersionString" variable must be set to -// Python.Runtime's current version. -#define USE_PYTHON_RUNTIME_VERSION - -// If defined, the "PythonRuntimePublicKeyTokenData" data array must be -// set to Python.Runtime's public key token. (sn -T Python.Runtin.dll) -#define USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - -// If DEBUG is defined in the Build Properties, a few Console.WriteLine -// calls are made to indicate what's going on during the load... -//============================================================================ -using System; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Reflection; -using System.Runtime.InteropServices; -using NXPorts.Attributes; - -public class clrModule -{ - [DllExport("PyInit_clr", CallingConvention.StdCall)] - public static IntPtr PyInit_clr() - { - DebugPrint("Attempting to load 'Python.Runtime' using standard binding rules."); -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - var pythonRuntimePublicKeyTokenData = new byte[] { 0x50, 0x00, 0xfe, 0xa6, 0xcb, 0xa7, 0x02, 0xdd }; -#endif - - // Attempt to find and load Python.Runtime using standard assembly binding rules. - // This roughly translates into looking in order: - // - GAC - // - ApplicationBase - // - A PrivateBinPath under ApplicationBase - // With an unsigned assembly, the GAC is skipped. - var pythonRuntimeName = new AssemblyName("Python.Runtime") - { -#if USE_PYTHON_RUNTIME_VERSION - // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.5.0"), -#endif - CultureInfo = CultureInfo.InvariantCulture - }; -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - pythonRuntimeName.SetPublicKeyToken(pythonRuntimePublicKeyTokenData); -#endif - // We've got the AssemblyName with optional features; try to load it. - Assembly pythonRuntime; - try - { - pythonRuntime = Assembly.Load(pythonRuntimeName); - DebugPrint("Success loading 'Python.Runtime' using standard binding rules."); - } - catch (IOException) - { - DebugPrint("'Python.Runtime' not found using standard binding rules."); - try - { - // If the above fails for any reason, we fallback to attempting to load "Python.Runtime.dll" - // from the directory this assembly is running in. "This assembly" is probably "clr.pyd", - // sitting somewhere in PYTHONPATH. This is using Assembly.LoadFrom, and inherits all the - // caveats of that call. See MSDN docs for details. - // Suzanne Cook's blog is also an excellent source of info on this: - // http://blogs.msdn.com/suzcook/ - // http://blogs.msdn.com/suzcook/archive/2003/05/29/57143.aspx - // http://blogs.msdn.com/suzcook/archive/2003/06/13/57180.aspx - - Assembly executingAssembly = Assembly.GetExecutingAssembly(); - string assemblyDirectory = Path.GetDirectoryName(executingAssembly.Location); - if (assemblyDirectory == null) - { - throw new InvalidOperationException(executingAssembly.Location); - } - string pythonRuntimeDllPath = Path.Combine(assemblyDirectory, "Python.Runtime.dll"); - DebugPrint($"Attempting to load Python.Runtime from: '{pythonRuntimeDllPath}'."); - pythonRuntime = Assembly.LoadFrom(pythonRuntimeDllPath); - DebugPrint($"Success loading 'Python.Runtime' from: '{pythonRuntimeDllPath}'."); - } - catch (InvalidOperationException) - { - DebugPrint("Could not load 'Python.Runtime'."); - return IntPtr.Zero; - } - } - - // Once here, we've successfully loaded SOME version of Python.Runtime - // So now we get the PythonEngine and execute the InitExt method on it. - Type pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); - - return (IntPtr)pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); - } - - /// - /// Substitute for Debug.Writeline(...). Ideally we would use Debug.Writeline - /// but haven't been able to configure the TRACE from within Python. - /// - [Conditional("DEBUG")] - private static void DebugPrint(string str) - { - Console.WriteLine(str); - } -} diff --git a/src/clrmodule/Properties/AssemblyInfo.cs b/src/clrmodule/Properties/AssemblyInfo.cs deleted file mode 100644 index 5e2e05ed4..000000000 --- a/src/clrmodule/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System.Reflection; -using System.Runtime.InteropServices; - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("ae10d6a4-55c2-482f-9716-9988e6c169e3")] diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj deleted file mode 100644 index 8595fd0ba..000000000 --- a/src/clrmodule/clrmodule.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - net472 - win-x86;win-x64 - clr - - - - - - - 1.0.0 - all - runtime; build; native; contentfiles; analyzers - - - - - x86 - - - x64 - - diff --git a/src/console/pythonconsole.cs b/src/console/pythonconsole.cs index 912e9bb0d..bf17848f7 100644 --- a/src/console/pythonconsole.cs +++ b/src/console/pythonconsole.cs @@ -26,7 +26,7 @@ private PythonConsole() [STAThread] public static int Main(string[] args) { - // Only net40 is capable to safely inject python.runtime.dll into resources. + // Only .NET Framework is capable to safely inject python.runtime.dll into resources. #if NET40 // reference the static assemblyLoader to stop it being optimized away AssemblyLoader a = assemblyLoader; diff --git a/src/domain_tests/conftest.py b/src/domain_tests/conftest.py deleted file mode 100644 index 5f0d52e10..000000000 --- a/src/domain_tests/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -from subprocess import check_call -# test_proj_path = os.path.join(cwd, "..", "testing") -cfd = os.path.dirname(__file__) -bin_path = os.path.join(cfd, 'bin') -check_call(["dotnet", "build", cfd, '-o', bin_path]) \ No newline at end of file diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py deleted file mode 100644 index e24eb6976..000000000 --- a/src/domain_tests/test_domain_reload.py +++ /dev/null @@ -1,100 +0,0 @@ -import subprocess -import os -import platform - -import pytest - -def _run_test(testname): - dirname = os.path.split(__file__)[0] - exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') - args = [exename, testname] - - if platform.system() != 'Windows': - args = ['mono'] + args - - proc = subprocess.Popen(args) - proc.wait() - - assert proc.returncode == 0 - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_class(): - _run_test('class_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_class_member_static_function(): - _run_test('static_member_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_class_member_function(): - _run_test('member_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_class_member_field(): - _run_test('field_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_class_member_property(): - _run_test('property_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_rename_namespace(): - _run_test('namespace_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_field_visibility_change(): - _run_test("field_visibility_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_method_visibility_change(): - _run_test("method_visibility_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_property_visibility_change(): - _run_test("property_visibility_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_class_visibility_change(): - _run_test("class_visibility_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_method_parameters_change(): - _run_test("method_parameters_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_method_return_type_change(): - _run_test("method_return_type_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_field_type_change(): - _run_test("field_type_change") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -@pytest.mark.xfail(reason="Events not yet serializable") -def test_rename_event(): - _run_test('event_rename') - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") -def test_construct_removed_class(): - _run_test("construct_removed_class") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_out_to_ref_param(): - _run_test("out_to_ref_param") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_ref_to_out_param(): - _run_test("ref_to_out_param") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_ref_to_in_param(): - _run_test("ref_to_in_param") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_in_to_ref_param(): - _run_test("in_to_ref_param") - -@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') -def test_nested_type(): - _run_test("nested_type") diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 18fcd32d1..f0c00a6d8 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -1,33 +1,39 @@ namespace Python.EmbeddingTest { using System; using System.Collections.Generic; - using System.Text; + using System.Linq; using NUnit.Framework; using Python.Runtime; using Python.Runtime.Codecs; - public class Codecs { + public class Codecs + { [SetUp] - public void SetUp() { + public void SetUp() + { PythonEngine.Initialize(); } [TearDown] - public void Dispose() { + public void Dispose() + { PythonEngine.Shutdown(); } [Test] - public void ConversionsGeneric() { - ConversionsGeneric, ValueTuple>(); + public void TupleConversionsGeneric() + { + TupleConversionsGeneric, ValueTuple>(); } - static void ConversionsGeneric() { + static void TupleConversionsGeneric() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(T value) => restored = value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -38,15 +44,18 @@ static void ConversionsGeneric() { } [Test] - public void ConversionsObject() { - ConversionsObject, ValueTuple>(); + public void TupleConversionsObject() + { + TupleConversionsObject, ValueTuple>(); } - static void ConversionsObject() { + static void TupleConversionsObject() + { TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; using (Py.GIL()) - using (var scope = Py.CreateScope()) { + using (var scope = Py.CreateScope()) + { void Accept(object value) => restored = (T)value; var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); @@ -57,12 +66,15 @@ static void ConversionsObject() { } [Test] - public void TupleRoundtripObject() { + public void TupleRoundtripObject() + { TupleRoundtripObject, ValueTuple>(); } - static void TupleRoundtripObject() { + static void TupleRoundtripObject() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); Assert.AreEqual(expected: tuple, actual: restored); @@ -70,18 +82,305 @@ static void TupleRoundtripObject() { } [Test] - public void TupleRoundtripGeneric() { + public void TupleRoundtripGeneric() + { TupleRoundtripGeneric, ValueTuple>(); } - static void TupleRoundtripGeneric() { + static void TupleRoundtripGeneric() + { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) { + using (Py.GIL()) + { var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); Assert.AreEqual(expected: tuple, actual: restored); } } + + static PyObject GetPythonIterable() + { + using (Py.GIL()) + { + return PythonEngine.Eval("map(lambda x: x, [1,2,3])"); + } + } + + [Test] + public void ListDecoderTest() + { + var codec = ListDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + + var pyListType = pyList.GetPythonType(); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //we'd have to copy into a list instance to do this, it would not be lossless. + //lossy converters can be implemented outside of the python.net core library + Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); + + //convert to list of int + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); + CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); + + //convert to list of string. This will not work. + //The ListWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python list can be queried without any conversion, + //the IList will report a Count of 3. + IList stringList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); + Assert.AreEqual(stringList.Count, 3); + Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; }); + + //can't convert python iterable to list (this will require a copy which isn't lossless) + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); + } + + [Test] + public void SequenceDecoderTest() + { + var codec = SequenceDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + //SequenceConverter can only convert to any ICollection + var pyList = new PyList(items.ToArray()); + //it can convert a PyList, since PyList satisfies the python sequence protocol + + Assert.IsFalse(codec.CanDecode(pyList, typeof(bool))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable))); + + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); + CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); + Assert.AreEqual(3, stringCollection.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + //can't convert python iterable to collection (this will require a copy which isn't lossless) + //python iterables do not satisfy the python sequence protocol + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); + + //python tuples do satisfy the python sequence protocol + var pyTuple = new PyTuple(items.ToArray()); + var pyTupleType = pyTuple.GetPythonType(); + + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); + CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); + Assert.AreEqual(3, stringCollection2.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection2.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + } + + [Test] + public void IterableDecoderTest() + { + var codec = IterableDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + var pyListType = pyList.GetPythonType(); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); + CollectionAssert.AreEqual(plainEnumerable1, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will lead to an empty iterable when decoding. TODO - should it throw? + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); + CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); + + Assert.Throws(typeof(InvalidCastException), () => { + foreach (string item in stringEnumerable) + { + var x = item; + } + }); + Assert.Throws(typeof(InvalidCastException), () => { + stringEnumerable.Count(); + }); + + Runtime.CheckExceptionOccurred(); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); + CollectionAssert.AreEqual(plainEnumerable2, new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + } + + // regression for https://github.com/pythonnet/pythonnet/issues/1427 + [Ignore("https://github.com/pythonnet/pythonnet/issues/1256")] + [Test] + public void PythonRegisteredDecoder_NoStackOverflowOnSystemType() + { + const string PyCode = @" +import clr +import System +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder + + +class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = 'Dummy' + def CanEncode(self, clr_type): + return clr_type.Name == 'IList`1' and clr_type.Namespace == 'System.Collections.Generic' + + +list_encoder = ListAsRawEncoder() +PyObjectConversions.RegisterEncoder(list_encoder) + +system_type = list_encoder.GetType()"; + + PythonEngine.Exec(PyCode); + } + + const string TestExceptionMessage = "Hello World!"; + [Test] + public void ExceptionEncoded() + { + PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); + void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); + var callMeAction = new Action(CallMe); + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + scope.Exec(@" +def call(func): + try: + func() + except ValueError as e: + return str(e) +"); + var callFunc = scope.Get("call"); + string message = callFunc.Invoke(callMeAction.ToPython()).As(); + Assert.AreEqual(TestExceptionMessage, message); + } + + [Test] + public void ExceptionDecoded() + { + PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); + using var _ = Py.GIL(); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() + => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + + class ValueErrorWrapper : Exception + { + public ValueErrorWrapper(string message) : base(message) { } + } + + class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder + { + public bool CanDecode(PyObject objectType, Type targetType) + => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError")); + + public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) + || typeof(ValueErrorWrapper).IsSubclassOf(type); + + public bool TryDecode(PyObject pyObj, out T value) + { + var message = pyObj.GetAttr("args")[0].As(); + value = (T)(object)new ValueErrorWrapper(message); + return true; + } + + public PyObject TryEncode(object value) + { + var error = (ValueErrorWrapper)value; + return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); + } + } } /// diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs index 458ab6a99..9a832cb0c 100644 --- a/src/embed_tests/GlobalTestsSetup.cs +++ b/src/embed_tests/GlobalTestsSetup.cs @@ -7,7 +7,7 @@ namespace Python.EmbeddingTest // As the SetUpFixture, the OneTimeTearDown of this class is executed after // all tests have run. [SetUpFixture] - public class GlobalTestsSetup + public partial class GlobalTestsSetup { [OneTimeTearDown] public void FinalCleanup() diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 1bb4fed11..67a7d3338 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -2,6 +2,8 @@ net472;netcoreapp3.1 + ..\pythonnet.snk + true diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs index 1d29e85c7..417e743c0 100644 --- a/src/embed_tests/References.cs +++ b/src/embed_tests/References.cs @@ -23,7 +23,7 @@ public void Dispose() public void MoveToPyObject_SetsNull() { var dict = new PyDict(); - NewReference reference = Runtime.PyDict_Items(dict.Handle); + NewReference reference = Runtime.PyDict_Items(dict.Reference); try { Assert.IsFalse(reference.IsNull()); @@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull() public void CanBorrowFromNewReference() { var dict = new PyDict(); - NewReference reference = Runtime.PyDict_Items(dict.Handle); + NewReference reference = Runtime.PyDict_Items(dict.Reference); try { PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference)); diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs index 454c97578..6875fde01 100644 --- a/src/embed_tests/TestCallbacks.cs +++ b/src/embed_tests/TestCallbacks.cs @@ -24,7 +24,7 @@ public void TestNoOverloadException() { using (Py.GIL()) { dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); - Assert.AreEqual("TypeError", error.PythonTypeName); + Assert.AreEqual("TypeError", error.Type.Name); string expectedArgTypes = "()"; StringAssert.EndsWith(expectedArgTypes, error.Message); } diff --git a/src/embed_tests/TestCustomMarshal.cs b/src/embed_tests/TestCustomMarshal.cs index 5860857a3..99911bdb0 100644 --- a/src/embed_tests/TestCustomMarshal.cs +++ b/src/embed_tests/TestCustomMarshal.cs @@ -23,7 +23,7 @@ public static void GetManagedStringTwice() { const string expected = "FooBar"; - IntPtr op = Runtime.Runtime.PyUnicode_FromString(expected); + IntPtr op = Runtime.Runtime.PyString_FromString(expected); string s1 = Runtime.Runtime.GetManagedString(op); string s2 = Runtime.Runtime.GetManagedString(op); diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index f8445edb4..e4479da18 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName) // assembly (and Python .NET) to reside var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Soft); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL); // From now on use the Proxy to call into the new assembly theProxy.RunPython(); @@ -400,7 +400,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Reload); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); arg = caller.Execute(arg); @@ -418,7 +418,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call("InitPython", ShutdownMode.Reload); + theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); caller.Execute(arg); @@ -478,8 +478,9 @@ public static void RunPython() private static IntPtr _state; - public static void InitPython(ShutdownMode mode) + public static void InitPython(ShutdownMode mode, string dllName) { + PyRuntime.PythonDLL = dllName; PythonEngine.Initialize(mode: mode); _state = PythonEngine.BeginAllowThreads(); } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 46e2fcdf1..c040e6930 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -101,7 +101,17 @@ public void CollectOnShutdown() PythonEngine.Shutdown(); garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.IsEmpty(garbage); + + if (garbage.Count > 0) + { + PythonEngine.Initialize(); + string objects = string.Join("\n", garbage.Select(ob => + { + var obj = new PyObject(new BorrowedReference(ob)); + return $"{obj} [{obj.GetPythonType()}@{obj.Handle}]"; + })); + Assert.Fail("Garbage is not empty:\n" + objects); + } } [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] // ensure lack of references to obj @@ -173,7 +183,7 @@ public void SimpleTestMemory() bool oldState = Finalizer.Instance.Enable; try { - using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyModule gcModule = PyModule.Import("gc")) using (PyObject pyCollect = gcModule.GetAttr("collect")) { long span1 = CompareWithFinalizerOn(pyCollect, false); diff --git a/src/embed_tests/TestInterrupt.cs b/src/embed_tests/TestInterrupt.cs new file mode 100644 index 000000000..a40407782 --- /dev/null +++ b/src/embed_tests/TestInterrupt.cs @@ -0,0 +1,64 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInterrupt + { + private IntPtr _threadState; + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + _threadState = PythonEngine.BeginAllowThreads(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.EndAllowThreads(_threadState); + PythonEngine.Shutdown(); + } + + [Test] + public void InterruptTest() + { + long pythonThreadID = 0; + var asyncCall = Task.Factory.StartNew(() => + { + using (Py.GIL()) + { + Interlocked.Exchange(ref pythonThreadID, (long)PythonEngine.GetPythonThreadID()); + return PythonEngine.RunSimpleString(@" +import time + +while True: + time.sleep(0.2)"); + } + }); + + var timeout = Stopwatch.StartNew(); + while (Interlocked.Read(ref pythonThreadID) == 0) + { + Assert.Less(timeout.Elapsed, TimeSpan.FromSeconds(5), "thread ID was not assigned in time"); + } + + using (Py.GIL()) + { + int interruptReturnValue = PythonEngine.Interrupt((ulong)Interlocked.Read(ref pythonThreadID)); + Assert.AreEqual(1, interruptReturnValue); + } + + Assert.IsTrue(asyncCall.Wait(TimeSpan.FromSeconds(5)), "Async thread was not interrupted in time"); + + Assert.AreEqual(-1, asyncCall.Result); + } + } +} diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 03812c6fe..8efd16e02 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -9,7 +9,7 @@ using Python.Runtime; -namespace Python.EmbeddingPythonTest +namespace Python.EmbeddingTest { public class TestNativeTypeOffset { @@ -34,11 +34,12 @@ public void Dispose() public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); - string attributeName = "abiflags"; - if (sys.HasAttr(attributeName) && !string.IsNullOrEmpty(sys.GetAttr(attributeName).ToString())) + // We can safely ignore the "m" abi flag + var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; - Assert.NotNull(Type.GetType(typeName), $"{typeName} does not exist and sys.{attributeName} is not empty"); + Assert.NotNull(Type.GetType(typeName), $"{typeName} does not exist and sys.abiflags={abiflags}"); } } } diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index 8e9feb241..68a6e8e35 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -335,6 +335,14 @@ public void SymmetricalOperatorOverloads() "); } + [Test] + public void EnumOperator() + { + PythonEngine.Exec($@" +from System.IO import FileAccess +c = FileAccess.Read | FileAccess.Write"); + } + [Test] public void OperatorOverloadMissingArgument() { diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index 94e7026c7..906c8cb0d 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -95,7 +95,7 @@ public void StringBadCtor() var ex = Assert.Throws(() => a = new PyFloat(i)); - StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message); + StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } @@ -132,7 +132,7 @@ public void AsFloatBad() PyFloat a = null; var ex = Assert.Throws(() => a = PyFloat.AsFloat(s)); - StringAssert.StartsWith("ValueError : could not convert string to float", ex.Message); + StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 005ab466d..bd6cf23a1 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -128,7 +128,7 @@ public void TestCtorBadString() var ex = Assert.Throws(() => a = new PyInt(i)); - StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message); + StringAssert.StartsWith("invalid literal for int", ex.Message); Assert.IsNull(a); } @@ -161,7 +161,7 @@ public void TestAsIntBad() PyInt a = null; var ex = Assert.Throws(() => a = PyInt.AsInt(s)); - StringAssert.StartsWith("ValueError : invalid literal for int", ex.Message); + StringAssert.StartsWith("invalid literal for int", ex.Message); Assert.IsNull(a); } diff --git a/src/embed_tests/TestPyList.cs b/src/embed_tests/TestPyList.cs index e9acfbb45..eee129f2d 100644 --- a/src/embed_tests/TestPyList.cs +++ b/src/embed_tests/TestPyList.cs @@ -41,7 +41,7 @@ public void TestStringAsListType() var ex = Assert.Throws(() => t = PyList.AsList(i)); - Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message); + Assert.AreEqual("'int' object is not iterable", ex.Message); Assert.IsNull(t); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index 3c155f315..6d587d064 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -144,7 +144,7 @@ public void TestCtorBadString() var ex = Assert.Throws(() => a = new PyLong(i)); - StringAssert.StartsWith("ValueError : invalid literal", ex.Message); + StringAssert.StartsWith("invalid literal", ex.Message); Assert.IsNull(a); } @@ -177,7 +177,7 @@ public void TestAsLongBad() PyLong a = null; var ex = Assert.Throws(() => a = PyLong.AsLong(s)); - StringAssert.StartsWith("ValueError : invalid literal", ex.Message); + StringAssert.StartsWith("invalid literal", ex.Message); Assert.IsNull(a); } diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 0de436e35..669ecde0d 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -94,5 +94,24 @@ public void TestUnicode() PyObject actual = new PyString(expected); Assert.AreEqual(expected, actual.ToString()); } + + [Test] + public void TestUnicodeSurrogateToString() + { + var expected = "foo\ud83d\udc3c"; + var actual = PythonEngine.Eval("'foo\ud83d\udc3c'"); + Assert.AreEqual(4, actual.Length()); + Assert.AreEqual(expected, actual.ToString()); + } + + [Test] + public void TestUnicodeSurrogate() + { + const string expected = "foo\ud83d\udc3c"; // "foo🐼" + PyObject actual = new PyString(expected); + // python treats "foo🐼" as 4 characters, dotnet as 5 + Assert.AreEqual(4, actual.Length()); + Assert.AreEqual(expected, actual.ToString()); + } } } diff --git a/src/embed_tests/TestPyTuple.cs b/src/embed_tests/TestPyTuple.cs index 362251049..5d76116aa 100644 --- a/src/embed_tests/TestPyTuple.cs +++ b/src/embed_tests/TestPyTuple.cs @@ -104,7 +104,7 @@ public void TestPyTupleInvalidAppend() var ex = Assert.Throws(() => t.Concat(s)); - StringAssert.StartsWith("TypeError : can only concatenate tuple", ex.Message); + StringAssert.StartsWith("can only concatenate tuple", ex.Message); Assert.AreEqual(0, t.Length()); Assert.IsEmpty(t); } @@ -164,7 +164,7 @@ public void TestInvalidAsTuple() var ex = Assert.Throws(() => t = PyTuple.AsTuple(i)); - Assert.AreEqual("TypeError : 'int' object is not iterable", ex.Message); + Assert.AreEqual("'int' object is not iterable", ex.Message); Assert.IsNull(t); } } diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs new file mode 100644 index 000000000..a28fe00da --- /dev/null +++ b/src/embed_tests/TestPyType.cs @@ -0,0 +1,47 @@ +using System.Runtime.InteropServices; +using System.Text; + +using NUnit.Framework; + +using Python.Runtime; +using Python.Runtime.Native; + +namespace Python.EmbeddingTest +{ + public class TestPyType + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void CanCreateHeapType() + { + const string name = "nÁmæ"; + const string docStr = "dÁcæ"; + + using var doc = new StrPtr(docStr, Encoding.UTF8); + var spec = new TypeSpec( + name: name, + basicSize: Marshal.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), + slots: new TypeSpec.Slot[] { + new (TypeSlotID.tp_doc, doc.RawPointer), + }, + TypeFlags.Default | TypeFlags.HeapType + ); + + using var type = new PyType(spec); + Assert.AreEqual(name, type.GetAttr("__name__").As()); + Assert.AreEqual(name, type.Name); + Assert.AreEqual(docStr, type.GetAttr("__doc__").As()); + } + } +} diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index dcd539504..c6228f1b9 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -51,7 +51,7 @@ def fail(self): catch (PythonException e) { TestContext.Out.WriteLine(e.Message); - Assert.IsTrue(e.Message.Contains("ZeroDivisionError")); + Assert.IsTrue(e.Type.Name == "ZeroDivisionError"); } } diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 3ab0f8106..a4b28906c 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -30,31 +30,61 @@ public void TestMessage() var ex = Assert.Throws(() => foo = list[0]); - Assert.AreEqual("IndexError : list index out of range", ex.Message); + Assert.AreEqual("list index out of range", ex.Message); + Assert.IsNull(foo); + } + + [Test] + public void TestType() + { + var list = new PyList(); + PyObject foo = null; + + var ex = Assert.Throws(() => foo = list[0]); + + Assert.AreEqual("IndexError", ex.Type.Name); Assert.IsNull(foo); } [Test] public void TestNoError() { - var e = new PythonException(); // There is no PyErr to fetch - Assert.AreEqual("", e.Message); + // There is no PyErr to fetch + Assert.Throws(() => PythonException.FetchCurrentRaw()); + var currentError = PythonException.FetchCurrentOrNullRaw(); + Assert.IsNull(currentError); } [Test] - public void TestPythonErrorTypeName() + public void TestNestedExceptions() { try { - var module = PythonEngine.ImportModule("really____unknown___module"); - Assert.Fail("Unknown module should not be loaded"); + PythonEngine.Exec(@" +try: + raise Exception('inner') +except Exception as ex: + raise Exception('outer') from ex +"); } catch (PythonException ex) { - Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError")); + Assert.That(ex.InnerException, Is.InstanceOf()); + Assert.That(ex.InnerException.Message, Is.EqualTo("inner")); } } + [Test] + public void InnerIsEmptyWithNoCause() + { + var list = new PyList(); + PyObject foo = null; + + var ex = Assert.Throws(() => foo = list[0]); + + Assert.IsNull(ex.InnerException); + } + [Test] public void TestPythonExceptionFormat() { @@ -65,15 +95,22 @@ public void TestPythonExceptionFormat() } catch (PythonException ex) { - Assert.That(ex.Format(), Does.Contain("Traceback").And.Contains("(most recent call last):").And.Contains("ValueError: Error!")); - } - } + // Console.WriteLine($"Format: {ex.Format()}"); + // Console.WriteLine($"Stacktrace: {ex.StackTrace}"); + Assert.That( + ex.Format(), + Does.Contain("Traceback") + .And.Contains("(most recent call last):") + .And.Contains("ValueError: Error!") + ); - [Test] - public void TestPythonExceptionFormatNoError() - { - var ex = new PythonException(); - Assert.AreEqual(ex.StackTrace, ex.Format()); + // Check that the stacktrace is properly formatted + Assert.That( + ex.StackTrace, + Does.Not.StartWith("[") + .And.Not.Contain("\\n") + ); + } } [Test] @@ -81,7 +118,7 @@ public void TestPythonExceptionFormatNoTraceback() { try { - var module = PythonEngine.ImportModule("really____unknown___module"); + var module = PyModule.Import("really____unknown___module"); Assert.Fail("Unknown module should not be loaded"); } catch (PythonException ex) @@ -97,6 +134,7 @@ public void TestPythonExceptionFormatNormalized() try { PythonEngine.Exec("a=b\n"); + Assert.Fail("Exception should have been raised"); } catch (PythonException ex) { @@ -117,23 +155,30 @@ def __init__(self, val): Assert.IsTrue(scope.TryGet("TestException", out PyObject type)); PyObject str = "dummy string".ToPython(); - IntPtr typePtr = type.Handle; - IntPtr strPtr = str.Handle; - IntPtr tbPtr = Runtime.Runtime.None.Handle; - Runtime.Runtime.XIncref(typePtr); - Runtime.Runtime.XIncref(strPtr); - Runtime.Runtime.XIncref(tbPtr); + var typePtr = new NewReference(type.Reference); + var strPtr = new NewReference(str.Reference); + var tbPtr = new NewReference(Runtime.Runtime.None.Reference); Runtime.Runtime.PyErr_NormalizeException(ref typePtr, ref strPtr, ref tbPtr); - using (PyObject typeObj = new PyObject(typePtr), strObj = new PyObject(strPtr), tbObj = new PyObject(tbPtr)) - { - // the type returned from PyErr_NormalizeException should not be the same type since a new - // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typePtr); - // the message should now be the string from the throw exception during normalization - Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); - } + using var typeObj = typePtr.MoveToPyObject(); + using var strObj = strPtr.MoveToPyObject(); + using var tbObj = tbPtr.MoveToPyObject(); + // the type returned from PyErr_NormalizeException should not be the same type since a new + // exception was raised by initializing the exception + Assert.AreNotEqual(type.Handle, typeObj.Handle); + // the message should now be the string from the throw exception during normalization + Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } } + + [Test] + public void TestPythonException_Normalize_ThrowsWhenErrorSet() + { + Exceptions.SetError(Exceptions.TypeError, "Error!"); + var pythonException = PythonException.FetchCurrentRaw(); + Exceptions.SetError(Exceptions.TypeError, "Another error"); + Assert.Throws(() => pythonException.Normalize()); + Exceptions.Clear(); + } } } diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index cde5dd6fa..9fb2e8b22 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; -using Python.Runtime.Platform; namespace Python.EmbeddingTest { @@ -37,7 +36,7 @@ public static void Py_IsInitializedValue() public static void RefCountTest() { Runtime.Runtime.Py_Initialize(); - IntPtr op = Runtime.Runtime.PyUnicode_FromString("FooBar"); + IntPtr op = Runtime.Runtime.PyString_FromString("FooBar"); // New object RefCount should be one Assert.AreEqual(1, Runtime.Runtime.Refcount(op)); @@ -92,23 +91,29 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() { Runtime.Runtime.Py_Initialize(); - // Create an instance of threading.Lock, which is one of the very few types that does not have the - // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. - var threading = Runtime.Runtime.PyImport_ImportModule("threading"); - Exceptions.ErrorCheck(threading); - var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); - Exceptions.ErrorCheck(threadingDict); - var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); - if (lockType == IntPtr.Zero) - throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); - - var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0)); - Exceptions.ErrorCheck(lockInstance); - - Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); - Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); - - Runtime.Runtime.Py_Finalize(); + try + { + // Create an instance of threading.Lock, which is one of the very few types that does not have the + // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. + using var threading = Runtime.Runtime.PyImport_ImportModule("threading"); + Exceptions.ErrorCheck(threading); + var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + Exceptions.ErrorCheck(threadingDict); + var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); + if (lockType.IsNull) + throw PythonException.ThrowLastAsClrException(); + + using var args = NewReference.DangerousFromPointer(Runtime.Runtime.PyTuple_New(0)); + using var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, args); + Exceptions.ErrorCheck(lockInstance); + + Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); + Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); + } + finally + { + Runtime.Runtime.Py_Finalize(); + } } } } diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index 43155e1bf..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using Python.Runtime.Platform; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - Runtime.Runtime.Shutdown(); - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); - Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = NativeCodePageHelper.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - /* if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } */ - } - } -} diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index 24f31acff..de8a06bf8 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -34,8 +34,10 @@ public void SetUp() TestContext.Out.WriteLine(testPath); IntPtr str = Runtime.Runtime.PyString_FromString(testPath); + Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); - Runtime.Runtime.PyList_Append(path, str); + Assert.IsFalse(path.IsNull); + Runtime.Runtime.PyList_Append(path, new BorrowedReference(str)); Runtime.Runtime.XDecref(str); } @@ -52,7 +54,7 @@ public void Dispose() [Test] public void TestDottedName() { - PyObject module = PythonEngine.ImportModule("PyImportTest.test.one"); + var module = PyModule.Import("PyImportTest.test.one"); Assert.IsNotNull(module); } @@ -62,7 +64,7 @@ public void TestDottedName() [Test] public void TestSysArgsImportException() { - PyObject module = PythonEngine.ImportModule("PyImportTest.sysargv"); + var module = PyModule.Import("PyImportTest.sysargv"); Assert.IsNotNull(module); } @@ -100,8 +102,7 @@ import clr clr.AddReference('{path}') "; - var error = Assert.Throws(() => PythonEngine.Exec(code)); - Assert.AreEqual(nameof(FileLoadException), error.PythonTypeName); + Assert.Throws(() => PythonEngine.Exec(code)); } } } diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index c774680dd..1622f46d3 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -158,9 +158,10 @@ public static void TestRunExitFuncs() catch (PythonException e) { string msg = e.ToString(); + bool isImportError = e.Is(Exceptions.ImportError); Runtime.Runtime.Shutdown(); - if (e.IsMatches(Exceptions.ImportError)) + if (isImportError) { Assert.Ignore("no atexit module"); } @@ -175,7 +176,8 @@ public static void TestRunExitFuncs() { called = true; }; - atexit.InvokeMethod("register", callback.ToPython()); + atexit.InvokeMethod("register", callback.ToPython()).Dispose(); + atexit.Dispose(); Runtime.Runtime.Shutdown(); Assert.True(called); } diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c deleted file mode 100644 index cdfd89342..000000000 --- a/src/monoclr/clrmod.c +++ /dev/null @@ -1,215 +0,0 @@ -// #define Py_LIMITED_API 0x03050000 -#include - -#include "stdlib.h" - -#define MONO_VERSION "v4.0.30319.1" -#define MONO_DOMAIN "Python" - -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include "dirent.h" -#include "dlfcn.h" -#include "libgen.h" -#include "alloca.h" -#endif - -typedef struct -{ - MonoDomain *domain; - MonoAssembly *pr_assm; - MonoMethod *shutdown; - const char *pr_file; - char *error; - char *init_name; - char *shutdown_name; - PyObject *module; -} PyNet_Args; - -PyNet_Args *PyNet_Init(void); -static PyNet_Args *pn_args; - -PyMODINIT_FUNC -PyInit_clr(void) -{ - pn_args = PyNet_Init(); - if (pn_args->error) - { - return NULL; - } - - return pn_args->module; -} - -void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(PyNet_Args *user_data); - -// initialize Mono and PythonNet -PyNet_Args *PyNet_Init() -{ - PyObject *pn_module; - PyObject *pn_path; - PyNet_Args *pn_args; - pn_args = (PyNet_Args *)malloc(sizeof(PyNet_Args)); - - pn_module = PyImport_ImportModule("pythonnet"); - if (pn_module == NULL) - { - pn_args->error = "Failed to import pythonnet"; - return pn_args; - } - - pn_path = PyObject_CallMethod(pn_module, "get_assembly_path", NULL); - if (pn_path == NULL) - { - Py_DecRef(pn_module); - pn_args->error = "Failed to get assembly path"; - return pn_args; - } - - pn_args->pr_file = PyUnicode_AsUTF8(pn_path); - pn_args->error = NULL; - pn_args->shutdown = NULL; - pn_args->module = NULL; - -#ifdef __linux__ - // Force preload libmono-2.0 as global. Without this, on some systems - // symbols from libmono are not found by libmononative (which implements - // some of the System.* namespaces). Since the only happened on Linux so - // far, we hardcode the library name, load the symbols into the global - // namespace and leak the handle. - dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); -#endif - - pn_args->init_name = "Python.Runtime:InitExt()"; - pn_args->shutdown_name = "Python.Runtime:Shutdown()"; - - pn_args->domain = mono_jit_init_version(MONO_DOMAIN, MONO_VERSION); - - // XXX: Skip setting config for now, should be derived from pr_file - // mono_domain_set_config(pn_args->domain, ".", "Python.Runtime.dll.config"); - - /* - * Load the default Mono configuration file, this is needed - * if you are planning on using the dllmaps defined on the - * system configuration - */ - mono_config_parse(NULL); - - main_thread_handler(pn_args); - - if (pn_args->error != NULL) - { - PyErr_SetString(PyExc_ImportError, pn_args->error); - } - return pn_args; -} - -char *PyNet_ExceptionToString(MonoObject *e); - -// Shuts down PythonNet and cleans up Mono -void PyNet_Finalize(PyNet_Args *pn_args) -{ - MonoObject *exception = NULL; - - if (pn_args->shutdown) - { - mono_runtime_invoke(pn_args->shutdown, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - } - pn_args->shutdown = NULL; - } - - if (pn_args->domain) - { - mono_jit_cleanup(pn_args->domain); - pn_args->domain = NULL; - } - free(pn_args); -} - -MonoMethod *getMethodFromClass(MonoClass *cls, char *name) -{ - MonoMethodDesc *mdesc; - MonoMethod *method; - - mdesc = mono_method_desc_new(name, 1); - method = mono_method_desc_search_in_class(mdesc, cls); - mono_method_desc_free(mdesc); - - return method; -} - -void main_thread_handler(PyNet_Args *user_data) -{ - PyNet_Args *pn_args = user_data; - MonoMethod *init; - MonoImage *pr_image; - MonoClass *pythonengine; - MonoObject *exception = NULL; - MonoObject *init_result; - - pn_args->pr_assm = mono_domain_assembly_open(pn_args->domain, pn_args->pr_file); - if (!pn_args->pr_assm) - { - pn_args->error = "Unable to load assembly"; - return; - } - - pr_image = mono_assembly_get_image(pn_args->pr_assm); - if (!pr_image) - { - pn_args->error = "Unable to get image"; - return; - } - - pythonengine = mono_class_from_name(pr_image, "Python.Runtime", "PythonEngine"); - if (!pythonengine) - { - pn_args->error = "Unable to load class PythonEngine from Python.Runtime"; - return; - } - - init = getMethodFromClass(pythonengine, pn_args->init_name); - if (!init) - { - pn_args->error = "Unable to fetch Init method from PythonEngine"; - return; - } - - pn_args->shutdown = getMethodFromClass(pythonengine, pn_args->shutdown_name); - if (!pn_args->shutdown) - { - pn_args->error = "Unable to fetch shutdown method from PythonEngine"; - return; - } - - init_result = mono_runtime_invoke(init, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - return; - } - - pn_args->module = *(PyObject**)mono_object_unbox(init_result); -} - -// Get string from a Mono exception -char *PyNet_ExceptionToString(MonoObject *e) -{ - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); - MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); - mono_method_desc_free(mdesc); - - mmethod = mono_object_get_virtual_method(e, mmethod); - MonoString *monoString = (MonoString*) mono_runtime_invoke(mmethod, e, NULL, NULL); - mono_runtime_invoke(mmethod, e, NULL, NULL); - return mono_string_to_utf8(monoString); -} diff --git a/src/python_tests_runner/Python.PythonTestsRunner.csproj b/src/python_tests_runner/Python.PythonTestsRunner.csproj index 2d6544614..1006b2148 100644 --- a/src/python_tests_runner/Python.PythonTestsRunner.csproj +++ b/src/python_tests_runner/Python.PythonTestsRunner.csproj @@ -6,6 +6,7 @@ + diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 79b15700e..36e8049d4 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -8,6 +8,7 @@ using NUnit.Framework; using Python.Runtime; +using Python.Test; namespace Python.PythonTestsRunner { @@ -50,7 +51,7 @@ public void RunPythonTest(string testFile, string testName) { folder = Path.GetDirectoryName(folder); } - folder = Path.Combine(folder, "tests"); + folder = Path.Combine(folder, "..", "tests"); string path = Path.Combine(folder, testFile + ".py"); if (!File.Exists(path)) throw new FileNotFoundException("Cannot find test file", path); diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index a9ea327e9..bf8a91d3e 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,10 +10,13 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; - /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; + /// Gets a raw pointer to the Python object + public IntPtr DangerousGetAddressOrNull() => this.pointer; + + public static BorrowedReference Null => new BorrowedReference(); /// /// Creates new instance of from raw pointer. Unsafe. @@ -27,6 +30,14 @@ public BorrowedReference(IntPtr pointer) => a.pointer == b.pointer; public static bool operator !=(BorrowedReference a, BorrowedReference b) => a.pointer != b.pointer; + public static bool operator ==(BorrowedReference reference, NullOnly @null) + => reference.IsNull; + public static bool operator !=(BorrowedReference reference, NullOnly @null) + => !reference.IsNull; + public static bool operator ==(NullOnly @null, BorrowedReference reference) + => reference.IsNull; + public static bool operator !=(NullOnly @null, BorrowedReference reference) + => !reference.IsNull; public override bool Equals(object obj) { if (obj is IntPtr ptr) diff --git a/src/runtime/Codecs/EnumPyLongCodec.cs b/src/runtime/Codecs/EnumPyLongCodec.cs new file mode 100644 index 000000000..7dab98028 --- /dev/null +++ b/src/runtime/Codecs/EnumPyLongCodec.cs @@ -0,0 +1,68 @@ +using System; + +namespace Python.Runtime.Codecs +{ + [Obsolete] + public sealed class EnumPyLongCodec : IPyObjectEncoder, IPyObjectDecoder + { + public static EnumPyLongCodec Instance { get; } = new EnumPyLongCodec(); + + public bool CanDecode(PyObject objectType, Type targetType) + { + return targetType.IsEnum + && objectType.IsSubclass(new BorrowedReference(Runtime.PyLongType)); + } + + public bool CanEncode(Type type) + { + return type == typeof(object) || type == typeof(ValueType) || type.IsEnum; + } + + public bool TryDecode(PyObject pyObj, out T value) + { + value = default; + if (!typeof(T).IsEnum) return false; + + Type etype = Enum.GetUnderlyingType(typeof(T)); + + if (!PyLong.IsLongType(pyObj)) return false; + + object result; + try + { + result = pyObj.AsManagedObject(etype); + } + catch (InvalidCastException) + { + return false; + } + + if (Enum.IsDefined(typeof(T), result) || typeof(T).IsFlagsEnum()) + { + value = (T)Enum.ToObject(typeof(T), result); + return true; + } + + return false; + } + + public PyObject TryEncode(object value) + { + if (value is null) return null; + + var enumType = value.GetType(); + if (!enumType.IsEnum) return null; + + try + { + return new PyLong((long)value); + } + catch (InvalidCastException) + { + return new PyLong((ulong)value); + } + } + + private EnumPyLongCodec() { } + } +} diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs new file mode 100644 index 000000000..346057238 --- /dev/null +++ b/src/runtime/Codecs/IterableDecoder.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class IterableDecoder : IPyObjectDecoder + { + internal static bool IsIterable(Type targetType) + { + //if it is a plain IEnumerable, we can decode it using sequence protocol. + if (targetType == typeof(System.Collections.IEnumerable)) + return true; + + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IEnumerable<>); + } + + internal static bool IsIterable(PyObject objectType) + { + return objectType.HasAttr("__iter__"); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsIterable(objectType) && IsIterable(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + //first see if T is a plan IEnumerable + if (typeof(T) == typeof(System.Collections.IEnumerable)) + { + object enumerable = new CollectionWrappers.IterableWrapper(pyObj); + value = (T)enumerable; + return true; + } + + var elementType = typeof(T).GetGenericArguments()[0]; + var collectionType = typeof(CollectionWrappers.IterableWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static IterableDecoder Instance { get; } = new IterableDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs new file mode 100644 index 000000000..013f3f3f9 --- /dev/null +++ b/src/runtime/Codecs/ListDecoder.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class ListDecoder : IPyObjectDecoder + { + private static bool IsList(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(IList<>); + } + + private static bool IsList(PyObject objectType) + { + //TODO accept any python object that implements the sequence and list protocols + //must implement sequence protocol to fully implement list protocol + //if (!SequenceDecoder.IsSequence(objectType)) return false; + + //returns wheter the type is a list. + return objectType.Handle == Runtime.PyListType; + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsList(objectType) && IsList(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.ListWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static ListDecoder Instance { get; } = new ListDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs new file mode 100644 index 000000000..dce08fd99 --- /dev/null +++ b/src/runtime/Codecs/SequenceDecoder.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.Codecs +{ + public class SequenceDecoder : IPyObjectDecoder + { + internal static bool IsSequence(Type targetType) + { + if (!targetType.IsGenericType) + return false; + + return targetType.GetGenericTypeDefinition() == typeof(ICollection<>); + } + + internal static bool IsSequence(PyObject objectType) + { + //must implement iterable protocol to fully implement sequence protocol + if (!IterableDecoder.IsIterable(objectType)) return false; + + //returns wheter it implements the sequence protocol + //according to python doc this needs to exclude dict subclasses + //but I don't know how to look for that given the objectType + //rather than the instance. + return objectType.HasAttr("__getitem__"); + } + + public bool CanDecode(PyObject objectType, Type targetType) + { + return IsSequence(objectType) && IsSequence(targetType); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + var elementType = typeof(T).GetGenericArguments()[0]; + Type collectionType = typeof(CollectionWrappers.SequenceWrapper<>).MakeGenericType(elementType); + + var instance = Activator.CreateInstance(collectionType, new[] { pyObj }); + value = (T)instance; + return true; + } + + public static SequenceDecoder Instance { get; } = new SequenceDecoder(); + + public static void Register() + { + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs new file mode 100644 index 000000000..97979b59b --- /dev/null +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Collections; + +namespace Python.Runtime.CollectionWrappers +{ + internal class IterableWrapper : IEnumerable + { + protected readonly PyObject pyObject; + + public IterableWrapper(PyObject pyObj) + { + if (pyObj == null) + throw new ArgumentNullException(); + pyObject = new PyObject(pyObj.Reference); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public IEnumerator GetEnumerator() + { + PyObject iterObject = null; + using (Py.GIL()) + iterObject = new PyObject(Runtime.PyObject_GetIter(pyObject.Handle)); + + while (true) + { + using (Py.GIL()) + { + var item = Runtime.PyIter_Next(iterObject.Handle); + if (item == IntPtr.Zero) + { + Runtime.CheckExceptionOccurred(); + iterObject.Dispose(); + break; + } + + yield return (T)new PyObject(item).AsManagedObject(typeof(T)); + } + } + } + } +} diff --git a/src/runtime/CollectionWrappers/ListWrapper.cs b/src/runtime/CollectionWrappers/ListWrapper.cs new file mode 100644 index 000000000..ec2476370 --- /dev/null +++ b/src/runtime/CollectionWrappers/ListWrapper.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class ListWrapper : SequenceWrapper, IList + { + public ListWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public T this[int index] + { + get + { + var item = Runtime.PyList_GetItem(pyObject.Reference, index); + var pyItem = new PyObject(item); + return pyItem.As(); + } + set + { + var pyItem = value.ToPython(); + var result = Runtime.PyList_SetItem(pyObject.Handle, index, pyItem.Handle); + if (result == -1) + Runtime.CheckExceptionOccurred(); + } + } + + public int IndexOf(T item) + { + return indexOf(item); + } + + public void Insert(int index, T item) + { + if (IsReadOnly) + throw new InvalidOperationException("Collection is read-only"); + + var pyItem = item.ToPython(); + + var result = Runtime.PyList_Insert(pyObject.Reference, index, pyItem.Handle); + if (result == -1) + Runtime.CheckExceptionOccurred(); + } + + public void RemoveAt(int index) + { + var result = removeAt(index); + + //PySequence_DelItem will set an error if it fails. throw it here + //since RemoveAt does not have a bool return value. + if (result == false) + Runtime.CheckExceptionOccurred(); + } + } +} diff --git a/src/runtime/CollectionWrappers/SequenceWrapper.cs b/src/runtime/CollectionWrappers/SequenceWrapper.cs new file mode 100644 index 000000000..945019850 --- /dev/null +++ b/src/runtime/CollectionWrappers/SequenceWrapper.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; + +namespace Python.Runtime.CollectionWrappers +{ + internal class SequenceWrapper : IterableWrapper, ICollection + { + public SequenceWrapper(PyObject pyObj) : base(pyObj) + { + + } + + public int Count + { + get + { + var size = Runtime.PySequence_Size(pyObject.Reference); + if (size == -1) + { + Runtime.CheckExceptionOccurred(); + } + + return (int)size; + } + } + + public virtual bool IsReadOnly => false; + + public virtual void Add(T item) + { + //not implemented for Python sequence. + //ICollection is the closest analogue but it doesn't map perfectly. + //SequenceWrapper is not final and can be subclassed if necessary + throw new NotImplementedException(); + } + + public void Clear() + { + if (IsReadOnly) + throw new NotImplementedException(); + var result = Runtime.PySequence_DelSlice(pyObject.Handle, 0, Count); + if (result == -1) + { + Runtime.CheckExceptionOccurred(); + } + } + + public bool Contains(T item) + { + //not sure if IEquatable is implemented and this will work! + foreach (var element in this) + if (element.Equals(item)) return true; + + return false; + } + + public void CopyTo(T[] array, int arrayIndex) + { + if (array == null) + throw new NullReferenceException(); + + if ((array.Length - arrayIndex) < this.Count) + throw new InvalidOperationException("Attempting to copy to an array that is too small"); + + var index = 0; + foreach (var item in this) + { + array[index + arrayIndex] = item; + index++; + } + } + + protected bool removeAt(int index) + { + if (IsReadOnly) + throw new NotImplementedException(); + if (index >= Count || index < 0) + return false; + + var result = Runtime.PySequence_DelItem(pyObject.Handle, index); + + if (result == 0) + return true; + + Runtime.CheckExceptionOccurred(); + return false; + } + + protected int indexOf(T item) + { + var index = 0; + foreach (var element in this) + { + if (element.Equals(item)) return index; + index++; + } + + return -1; + } + + public bool Remove(T item) + { + var result = removeAt(indexOf(item)); + + //clear the python exception from PySequence_DelItem + //it is idiomatic in C# to return a bool rather than + //throw for a failed Remove in ICollection + if (result == false) + Runtime.PyErr_Clear(); + return result; + } + } +} diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 0cbbbaba2..3ef5cd662 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -41,8 +41,9 @@ public int GetNativeDataSize() /// internal class UcsMarshaler : MarshalerBase { + internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); - private static readonly Encoding PyEncoding = Runtime.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -91,13 +92,13 @@ public static int GetUnicodeByteLength(IntPtr p) var len = 0; while (true) { - int c = Runtime._UCS == 2 + int c = _UCS == 2 ? Marshal.ReadInt16(p, len * 2) : Marshal.ReadInt32(p, len * 4); if (c == 0) { - return len * Runtime._UCS; + return len * _UCS; } checked { @@ -147,7 +148,7 @@ public static string PtrToPy3UnicodePy2String(IntPtr p) internal class StrArrayMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrArrayMarshaler(); - private static readonly Encoding PyEncoding = Runtime.PyEncoding; + private static readonly Encoding PyEncoding = UcsMarshaler.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -159,7 +160,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) } int totalStrLength = argv.Sum(arg => arg.Length + 1); - int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS; + int memSize = argv.Length * IntPtr.Size + totalStrLength * UcsMarshaler._UCS; IntPtr mem = Marshal.AllocHGlobal(memSize); try @@ -188,49 +189,4 @@ public static ICustomMarshaler GetInstance(string cookie) return Instance; } } - - - /// - /// Custom Marshaler to deal with Managed String to Native - /// conversion on UTF-8. Use on functions that expect UTF-8 encoded - /// strings like `PyUnicode_FromStringAndSize` - /// - /// - /// If instead we used `MarshalAs(UnmanagedType.LPWStr)` the output to - /// `foo` would be `f\x00o\x00o\x00`. - /// - internal class Utf8Marshaler : MarshalerBase - { - private static readonly MarshalerBase Instance = new Utf8Marshaler(); - private static readonly Encoding PyEncoding = Encoding.UTF8; - - public override IntPtr MarshalManagedToNative(object managedObj) - { - var s = managedObj as string; - - if (s == null) - { - return IntPtr.Zero; - } - - byte[] bStr = PyEncoding.GetBytes(s + "\0"); - IntPtr mem = Marshal.AllocHGlobal(bStr.Length); - try - { - Marshal.Copy(bStr, 0, mem, bStr.Length); - } - catch (Exception) - { - Marshal.FreeHGlobal(mem); - throw; - } - - return mem; - } - - public static ICustomMarshaler GetInstance(string cookie) - { - return Instance; - } - } } diff --git a/src/runtime/ManagedTypes.cd b/src/runtime/ManagedTypes.cd new file mode 100644 index 000000000..385ae7117 --- /dev/null +++ b/src/runtime/ManagedTypes.cd @@ -0,0 +1,196 @@ + + + + + + FAAAAgAIAAAEDAAAAAAAAEACIACJAAIAAAAAAAIAAAQ= + classbase.cs + + + + + + AAAAAABIAAABDAAAIAIAAAAAAAAAAAAAAAAACAiAAAQ= + classderived.cs + + + + + + AAAAAAAAABAAAAAAAAAAACAAIAAJAAAAIAAAAACAAAI= + arrayobject.cs + + + + + + AAABAAAIAAAAAAAAIAAAAAAAAAAAAIAAACAAAACAAAA= + classobject.cs + + + + + + AAAACAAAAAAABABAAAAACAAAABAJAEAAAAAAAAIAAAA= + constructorbinding.cs + + + + + + EAAAAAAAAAAAAAAAAAACAAACBIAAAAJAAAAAAAAAAAA= + clrobject.cs + + + + + + AAAAAEAgIAQABAAAAABAAAAAIAIAAAAAAhAQAAAAKBA= + moduleobject.cs + + + + + + AAAACAAAAAAABAAAAAAACAAAABAJAEAAAAAAAAIAEAA= + constructorbinding.cs + + + + + + AAABAAAAAAAAAABAAAAAAEAAIAACAAAAAAAAAACAAAA= + delegateobject.cs + + + + + + + + + + + + + + AAAAAAAAAAAADAAAIAAAEABAAAAAAAACAAAAAAIAAAQ= + eventbinding.cs + + + + + + AAACAAAAAAAAAAAAAAIAAIAAAAAEAAAAQABAAAIBEAQ= + eventobject.cs + + + + + + AAAAAgAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + exceptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAACAAAAAEEBAAAAAAABAAQ= + extensiontype.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAIBEAA= + fieldobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAggAAAAAAAEAACAACAAAA= + interfaceobject.cs + + + + + + UCBBgoBAIUgAAAEAACAAsAACAgAIABIAQYAAACIYIBA= + managedtype.cs + + + + + + AQAAAAAICBAAAQBAAAABAAIAAgABAAABAAAAUBCAAAQ= + metatype.cs + + + + + + + + + + + + + + EAAAAAAAAIAADABAIAAAAAAAAAgBAAAAUgAAAAIAAAQ= + methodbinding.cs + + + + + + FIADAAAAAAAIBAAAIAAIAAAIAAgFAAAAUAAgAAIAEAQ= + methodobject.cs + + + + + + AAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAIAAAA= + modulefunctionobject.cs + + + + + + ECCCCkAAAAAABAAAAAABAAACAAAIAIIAEAAAAAIACAQ= + moduleobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + modulepropertyobject.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAQAAAAAIBEAg= + propertyobject.cs + + + + + + + + + + + + + + AAAAAAAAAAAAAAAAIAAAAAAAAAABAAAAAgAAAAIAAAQ= + overload.cs + + + + \ No newline at end of file diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index a4ed75918..c037f988f 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -2,6 +2,7 @@ namespace Python.Runtime { using System; using System.Diagnostics.Contracts; + using System.Runtime.CompilerServices; /// /// Represents a reference to a Python object, that is tracked by Python's reference counting. @@ -11,6 +12,16 @@ ref struct NewReference { IntPtr pointer; + /// Creates a pointing to the same object + public NewReference(BorrowedReference reference, bool canBeNull = false) + { + var address = canBeNull + ? reference.DangerousGetAddressOrNull() + : reference.DangerousGetAddress(); + Runtime.XIncref(address); + this.pointer = address; + } + [Pure] public static implicit operator BorrowedReference(in NewReference reference) => new BorrowedReference(reference.pointer); @@ -28,6 +39,16 @@ public PyObject MoveToPyObject() return result; } + /// Moves ownership of this instance to unmanged pointer + public IntPtr DangerousMoveToPointer() + { + if (this.IsNull()) throw new NullReferenceException(); + + var result = this.pointer; + this.pointer = IntPtr.Zero; + return result; + } + /// Moves ownership of this instance to unmanged pointer public IntPtr DangerousMoveToPointerOrNull() { @@ -36,6 +57,34 @@ public IntPtr DangerousMoveToPointerOrNull() return result; } + /// + /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + /// + public PyObject MoveToPyObjectOrNull() => this.IsNull() ? null : this.MoveToPyObject(); + /// + /// Call this method to move ownership of this reference to a Python C API function, + /// that steals reference passed to it. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StolenReference StealNullable() + { + IntPtr rawPointer = this.pointer; + this.pointer = IntPtr.Zero; + return new StolenReference(rawPointer); + } + + /// + /// Call this method to move ownership of this reference to a Python C API function, + /// that steals reference passed to it. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StolenReference Steal() + { + if (this.IsNull()) throw new NullReferenceException(); + + return this.StealNullable(); + } /// /// Removes this reference to a Python object, and sets it to null. /// diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 470488c02..3417bccc8 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -1,3 +1,3 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Python.EmbeddingTest")] +[assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index f18cf7a49..0311dbf9a 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -2,23 +2,42 @@ netstandard2.0 AnyCPU + 9.0 Python.Runtime Python.Runtime + pythonnet - https://github.com/pythonnet/pythonnet/blob/master/LICENSE + LICENSE https://github.com/pythonnet/pythonnet git python interop dynamic dlr Mono pinvoke https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico https://pythonnet.github.io/ + true + Python and CLR (.NET and Mono) cross-platform language interop + + true + true + snupkg + + ..\pythonnet.snk + true + 1591;NU1701 True + + true - $(DefineConstants);$(ConfiguredConstants) + ..\..\pythonnet\runtime + false + + + + @@ -29,5 +48,6 @@ + diff --git a/src/runtime/StolenReference.cs b/src/runtime/StolenReference.cs new file mode 100644 index 000000000..1130cff06 --- /dev/null +++ b/src/runtime/StolenReference.cs @@ -0,0 +1,46 @@ +namespace Python.Runtime +{ + using System; + using System.Diagnostics.Contracts; + + /// + /// Should only be used for the arguments of Python C API functions, that steal references, + /// and internal constructors. + /// + [NonCopyable] + readonly ref struct StolenReference + { + internal readonly IntPtr Pointer; + + internal StolenReference(IntPtr pointer) + { + Pointer = pointer; + } + + [Pure] + public static bool operator ==(in StolenReference reference, NullOnly @null) + => reference.Pointer == IntPtr.Zero; + [Pure] + public static bool operator !=(in StolenReference reference, NullOnly @null) + => reference.Pointer != IntPtr.Zero; + + [Pure] + public override bool Equals(object obj) + { + if (obj is IntPtr ptr) + return ptr == Pointer; + + return false; + } + + [Pure] + public override int GetHashCode() => Pointer.GetHashCode(); + } + + static class StolenReferenceExtensions + { + [Pure] + public static IntPtr DangerousGetAddressOrNull(this in StolenReference reference) + => reference.Pointer; + } +} diff --git a/src/runtime/TypeSpec.cs b/src/runtime/TypeSpec.cs new file mode 100644 index 000000000..87c0f94bc --- /dev/null +++ b/src/runtime/TypeSpec.cs @@ -0,0 +1,121 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Python.Runtime +{ + public class TypeSpec + { + public TypeSpec(string name, int basicSize, IEnumerable slots, TypeFlags flags, int itemSize = 0) + { + this.Name = name ?? throw new ArgumentNullException(nameof(name)); + this.BasicSize = basicSize; + this.Slots = slots.ToArray(); + this.Flags = flags; + this.ItemSize = itemSize; + } + public string Name { get; } + public int BasicSize { get; } + public int ItemSize { get; } + public TypeFlags Flags { get; } + public IReadOnlyList Slots { get; } + + [StructLayout(LayoutKind.Sequential)] + public struct Slot + { + public Slot(TypeSlotID id, IntPtr value) + { + ID = id; + Value = value; + } + + public TypeSlotID ID { get; } + public IntPtr Value { get; } + } + } + + public enum TypeSlotID : int + { + mp_ass_subscript = 3, + mp_length = 4, + mp_subscript = 5, + nb_absolute = 6, + nb_add = 7, + nb_and = 8, + nb_bool = 9, + nb_divmod = 10, + nb_float = 11, + nb_floor_divide = 12, + nb_index = 13, + nb_inplace_add = 14, + nb_inplace_and = 15, + nb_inplace_floor_divide = 16, + nb_inplace_lshift = 17, + nb_inplace_multiply = 18, + nb_inplace_or = 19, + nb_inplace_power = 20, + nb_inplace_remainder = 21, + nb_inplace_rshift = 22, + nb_inplace_subtract = 23, + nb_inplace_true_divide = 24, + nb_inplace_xor = 25, + nb_int = 26, + nb_invert = 27, + nb_lshift = 28, + nb_multiply = 29, + nb_negative = 30, + nb_or = 31, + nb_positive = 32, + nb_power = 33, + nb_remainder = 34, + nb_rshift = 35, + nb_subtract = 36, + nb_true_divide = 37, + nb_xor = 38, + sq_ass_item = 39, + sq_concat = 40, + sq_contains = 41, + sq_inplace_concat = 42, + sq_inplace_repeat = 43, + sq_item = 44, + sq_length = 45, + sq_repeat = 46, + tp_alloc = 47, + tp_base = 48, + tp_bases = 49, + tp_call = 50, + tp_clear = 51, + tp_dealloc = 52, + tp_del = 53, + tp_descr_get = 54, + tp_descr_set = 55, + tp_doc = 56, + tp_getattr = 57, + tp_getattro = 58, + tp_hash = 59, + tp_init = 60, + tp_is_gc = 61, + tp_iter = 62, + tp_iternext = 63, + tp_methods = 64, + tp_new = 65, + tp_repr = 66, + tp_richcompare = 67, + tp_setattr = 68, + tp_setattro = 69, + tp_str = 70, + tp_traverse = 71, + tp_members = 72, + tp_getset = 73, + tp_free = 74, + nb_matrix_multiply = 75, + nb_inplace_matrix_multiply = 76, + am_await = 77, + am_aiter = 78, + am_anext = 79, + /// New in 3.5 + tp_finalize = 80, + } +} diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 262e521a5..5c97c6dbf 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -54,7 +54,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) // create single dimensional array if (Runtime.PyInt_Check(op)) { - dimensions[0] = Runtime.PyLong_AsLongLong(op); + dimensions[0] = Runtime.PyLong_AsSignedSize_t(op); if (dimensions[0] == -1 && Exceptions.ErrorOccurred()) { Exceptions.Clear(); @@ -89,7 +89,7 @@ static NewReference CreateMultidimensional(Type elementType, long[] dimensions, return default; } - dimensions[dimIndex] = Runtime.PyLong_AsLongLong(dimObj); + dimensions[dimIndex] = Runtime.PyLong_AsSignedSize_t(dimObj); if (dimensions[dimIndex] == -1 && Exceptions.ErrorOccurred()) { Exceptions.RaiseTypeError("array constructor expects integer dimensions"); @@ -159,6 +159,10 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (rank == 1) { + if (!Runtime.PyInt_Check(idx)) + { + return RaiseIndexMustBeIntegerError(idx); + } index = Runtime.PyInt_AsLong(idx); if (Exceptions.ErrorOccurred()) @@ -199,6 +203,10 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, for (var i = 0; i < count; i++) { IntPtr op = Runtime.PyTuple_GetItem(idx, i); + if (!Runtime.PyInt_Check(op)) + { + return RaiseIndexMustBeIntegerError(op); + } index = Runtime.PyInt_AsLong(op); if (Exceptions.ErrorOccurred()) @@ -253,6 +261,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, if (rank == 1) { + if (!Runtime.PyInt_Check(idx)) + { + RaiseIndexMustBeIntegerError(idx); + return -1; + } index = Runtime.PyInt_AsLong(idx); if (Exceptions.ErrorOccurred()) @@ -291,6 +304,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, for (var i = 0; i < count; i++) { IntPtr op = Runtime.PyTuple_GetItem(idx, i); + if (!Runtime.PyInt_Check(op)) + { + RaiseIndexMustBeIntegerError(op); + return -1; + } index = Runtime.PyInt_AsLong(op); if (Exceptions.ErrorOccurred()) @@ -320,6 +338,11 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, return 0; } + private static IntPtr RaiseIndexMustBeIntegerError(IntPtr idx) + { + string tpName = Runtime.PyObject_GetTypeName(idx); + return Exceptions.RaiseTypeError($"array index has type {tpName}, expected an integer"); + } /// /// Implements __contains__ for array types. diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 0387d2dfc..d44f5f666 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -37,7 +37,6 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; internal static List pypath; - private AssemblyManager() { } @@ -312,6 +311,15 @@ public static bool IsValidNamespace(string name) return !string.IsNullOrEmpty(name) && namespaces.ContainsKey(name); } + /// + /// Returns an IEnumerable containing the namepsaces exported + /// by loaded assemblies in the current app domain. + /// + public static IEnumerable GetNamespaces () + { + return namespaces.Keys; + } + /// /// Returns list of assemblies that declare types in a given namespace /// diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 872501267..55f5c5b8f 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,9 +1,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Runtime.Serialization; namespace Python.Runtime { @@ -66,7 +63,16 @@ public virtual IntPtr type_subscript(IntPtr idx) if (target != null) { - Type t = target.MakeGenericType(types); + Type t; + try + { + // MakeGenericType can throw ArgumentException + t = target.MakeGenericType(types); + } + catch (ArgumentException e) + { + return Exceptions.RaiseTypeError(e.Message); + } ManagedType c = ClassManager.GetClass(t); Runtime.XIncref(c.pyHandle); return c.pyHandle; @@ -263,14 +269,14 @@ public static IntPtr tp_iter(IntPtr ob) /// /// Standard __hash__ implementation for instances of reflected types. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var co = GetManagedObject(ob) as CLRObject; if (co == null) { return Exceptions.RaiseTypeError("unhashable type"); } - return new IntPtr(co.inst.GetHashCode()); + return co.inst.GetHashCode(); } @@ -346,26 +352,28 @@ public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); tp_clear(ob); - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); - self.FreeGCHandle(); + Runtime.PyObject_GC_UnTrack(ob); + Runtime.PyObject_GC_Del(ob); + self?.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { ManagedType self = GetManagedObject(ob); - if (!self.IsTypeObject()) + + bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType; + if (!isTypeObject) { ClearObjectDict(ob); } - self.tpHandle = IntPtr.Zero; + if (self is not null) self.tpHandle = IntPtr.Zero; return 0; } protected override void OnSave(InterDomainContext context) { base.OnSave(context); - if (pyHandle != tpHandle) + if (!this.IsClrMetaTypeInstance()) { IntPtr dict = GetObjectDict(pyHandle); Runtime.XIncref(dict); @@ -376,13 +384,13 @@ protected override void OnSave(InterDomainContext context) protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); - if (pyHandle != tpHandle) + if (!this.IsClrMetaTypeInstance()) { IntPtr dict = context.Storage.GetValue("dict"); SetObjectDict(pyHandle, dict); } gcHandle = AllocGCHandle(); - Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + SetGCHandle(ObjectReference, gcHandle); } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 4e8e88bf3..cc2397225 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Reflection.Emit; @@ -75,8 +76,8 @@ internal ClassDerivedObject(Type tp) : base(tp) // So we don't call PyObject_GC_Del here and instead we set the python // reference to a weak reference so that the C# object can be collected. GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - int gcOffset = ObjectOffset.magic(Runtime.PyObject_TYPE(self.pyHandle)); - Marshal.WriteIntPtr(self.pyHandle, gcOffset, (IntPtr)gc); + Debug.Assert(self.TypeReference == Runtime.PyObject_TYPE(self.ObjectReference)); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; } @@ -101,12 +102,9 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { - -#if PYTHON_WITH_PYDEBUG - Runtime._Py_NewReference(self.pyHandle); -#endif + Runtime._Py_NewReference(self.ObjectReference); GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); - Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; @@ -124,11 +122,13 @@ internal static IntPtr ToPython(IPythonDerivedType obj) /// internal static Type CreateDerivedType(string name, Type baseType, - IntPtr py_dict, + BorrowedReference dictRef, string namespaceStr, string assemblyName, string moduleName = "Python.Runtime.Dynamic.dll") { + // TODO: clean up + IntPtr py_dict = dictRef.DangerousGetAddress(); if (null != namespaceStr) { name = namespaceStr + "." + name; @@ -826,7 +826,7 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec try { // create the python object - IntPtr type = TypeManager.GetTypeHandle(obj.GetType()); + BorrowedReference type = TypeManager.GetTypeReference(obj.GetType()); self = new CLRObject(obj, type); // set __pyobj__ to self and deref the python object which will allow this @@ -883,11 +883,6 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); - if (dict != IntPtr.Zero) - { - Runtime.XDecref(dict); - } Runtime.PyObject_GC_Del(self.pyHandle); self.gcHandle.Free(); } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0cbff371f..811b802c9 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -117,7 +117,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // Python object's dictionary tool; thus raising an AttributeError // instead of a TypeError. // Classes are re-initialized on in RestoreRuntimeData. - IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + using var dict = Runtime.PyObject_GenericGetDict(cls.Value.TypeReference); foreach (var member in cls.Value.dotNetMembers) { // No need to decref the member, the ClassBase instance does @@ -131,11 +131,11 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) } else if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } // We modified the Type object, notify it we did. - Runtime.PyType_Modified(cls.Value.tpHandle); + Runtime.PyType_Modified(cls.Value.TypeReference); } } @@ -155,7 +155,7 @@ internal static Dictionary RestoreRuntimeData(R // re-init the class InitClassBase(pair.Key.Value, pair.Value); // We modified the Type object, notify it we did. - Runtime.PyType_Modified(pair.Value.tpHandle); + Runtime.PyType_Modified(pair.Value.TypeReference); var context = contexts[pair.Value.pyHandle]; pair.Value.Load(context); loadedObjs.Add(pair.Value, context); @@ -266,10 +266,10 @@ private static void InitClassBase(Type type, ClassBase impl) // point to the managed methods providing the implementation. - IntPtr tp = TypeManager.GetTypeHandle(impl, type); + var pyType = TypeManager.GetType(impl, type); // Finally, initialize the class __dict__ and return the object. - IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + using var dict = Runtime.PyObject_GenericGetDict(pyType.Reference); if (impl.dotNetMembers == null) @@ -282,7 +282,7 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; impl.dotNetMembers.Add(name); - Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + Runtime.PyDict_SetItemString(dict, name, item.ObjectReference); // Decref the item now that it's been used. item.DecrRefCount(); if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp)) { @@ -291,20 +291,15 @@ private static void InitClassBase(Type type, ClassBase impl) } // If class has constructors, generate an __doc__ attribute. - IntPtr doc = IntPtr.Zero; + NewReference doc = default; Type marker = typeof(DocStringAttribute); var attrs = (Attribute[])type.GetCustomAttributes(marker, false); - if (attrs.Length == 0) - { - doc = IntPtr.Zero; - } - else + if (attrs.Length != 0) { var attr = (DocStringAttribute)attrs[0]; string docStr = attr.DocString; - doc = Runtime.PyString_FromString(docStr); + doc = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docStr)); Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); - Runtime.XDecref(doc); } var co = impl as ClassObject; @@ -317,26 +312,27 @@ private static void InitClassBase(Type type, ClassBase impl) // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) { - var ctors = new ConstructorBinding(type, tp, co.binder); + var ctors = new ConstructorBinding(type, pyType, co.binder); // ExtensionType types are untracked, so don't Incref() them. // TODO: deprecate __overloads__ soon... - Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.pyHandle); - Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.pyHandle); + Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.ObjectReference); + Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.ObjectReference); ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. - if (!CLRModule._SuppressDocs && doc == IntPtr.Zero) + if (!CLRModule._SuppressDocs && doc.IsNull()) { doc = co.GetDocString(); Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc); - Runtime.XDecref(doc); } } } + doc.Dispose(); + // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(tp); + Runtime.PyType_Modified(pyType.Reference); } internal static bool ShouldBindMethod(MethodBase mb) @@ -407,6 +403,17 @@ private static ClassInfo GetClassInfo(Type type) } } + // only [Flags] enums support bitwise operations + if (type.IsEnum && type.IsFlagsEnum()) + { + var opsImpl = typeof(EnumOps<>).MakeGenericType(type); + foreach (var op in opsImpl.GetMethods(OpsHelper.BindingFlags)) + { + local[op.Name] = 1; + } + info = info.Concat(opsImpl.GetMethods(OpsHelper.BindingFlags)).ToArray(); + } + // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) { diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 826ae5c54..1a2532044 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -31,7 +31,7 @@ internal ClassObject(Type tp) : base(tp) /// /// Helper to get docstring from reflected constructor info. /// - internal IntPtr GetDocString() + internal NewReference GetDocString() { MethodBase[] methods = binder.GetMethods(); var str = ""; @@ -43,15 +43,16 @@ internal IntPtr GetDocString() } str += t.ToString(); } - return Runtime.PyString_FromString(str); + return NewReference.DangerousFromPointer(Runtime.PyString_FromString(str)); } /// /// Implements __new__ for reflected classes and value types. /// - public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) + public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) { + var tp = new BorrowedReference(tpRaw); var self = GetManagedObject(tp) as ClassObject; // Sanity check: this ensures a graceful error if someone does @@ -87,7 +88,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - return CLRObject.GetInstHandle(result, tp); + return CLRObject.GetInstHandle(result, tp).DangerousMoveToPointerOrNull(); } if (type.IsAbstract) @@ -98,8 +99,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) if (type.IsEnum) { - Exceptions.SetError(Exceptions.TypeError, "cannot instantiate enumeration"); - return IntPtr.Zero; + return NewEnum(type, new BorrowedReference(args), tp).DangerousMoveToPointerOrNull(); } object obj = self.binder.InvokeRaw(IntPtr.Zero, args, kw); @@ -108,7 +108,44 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - return CLRObject.GetInstHandle(obj, tp); + return CLRObject.GetInstHandle(obj, tp).DangerousMoveToPointerOrNull(); + } + + private static NewReference NewEnum(Type type, BorrowedReference args, BorrowedReference tp) + { + nint argCount = Runtime.PyTuple_Size(args); + bool allowUnchecked = false; + if (argCount == 2) + { + var allow = Runtime.PyTuple_GetItem(args, 1); + if (!Converter.ToManaged(allow, typeof(bool), out var allowObj, true) || allowObj is null) + { + Exceptions.RaiseTypeError("second argument to enum constructor must be a boolean"); + return default; + } + allowUnchecked |= (bool)allowObj; + } + + if (argCount < 1 || argCount > 2) + { + Exceptions.SetError(Exceptions.TypeError, "no constructors match given arguments"); + return default; + } + + var op = Runtime.PyTuple_GetItem(args, 0); + if (!Converter.ToManaged(op, type.GetEnumUnderlyingType(), out object result, true)) + { + return default; + } + + if (!allowUnchecked && !Enum.IsDefined(type, result) && !type.IsFlagsEnum()) + { + Exceptions.SetError(Exceptions.ValueError, "Invalid enumeration value. Pass True as the second argument if unchecked conversion is desired"); + return default; + } + + object enumValue = Enum.ToObject(type, result); + return CLRObject.GetInstHandle(enumValue, tp); } diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0a352b381..02f7baf65 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,28 +14,20 @@ internal CLRObject(object ob, IntPtr tp) System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Subclass) != 0) - { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); - if (dict == IntPtr.Zero) - { - dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); - } - } - - GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; inst = ob; + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + InitGCHandle(ObjectReference, type: TypeReference, gc); + // Fix the BaseException args (and __cause__ in case of Python 3) // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + if (ob is Exception e) Exceptions.SetArgsAndCause(ObjectReference, e); } + internal CLRObject(object ob, BorrowedReference tp) : this(ob, tp.DangerousGetAddress()) { } + protected CLRObject() { } @@ -78,6 +70,9 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } + internal static NewReference GetReference(object ob) + => NewReference.DangerousFromPointer(GetInstHandle(ob)); + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) { CLRObject co = new CLRObject() @@ -101,7 +96,7 @@ protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + SetGCHandle(ObjectReference, TypeReference, gc); } } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 803823e39..9ac1adc0f 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -23,16 +23,16 @@ namespace Python.Runtime internal class ConstructorBinding : ExtensionType { private MaybeType type; // The managed Type being wrapped in a ClassObject - private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. + private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; [NonSerialized] private IntPtr repr; - public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) + public ConstructorBinding(Type type, PyType typeToCreate, ConstructorBinder ctorBinder) { this.type = type; - this.pyTypeHndl = pyTypeHndl; // steal a type reference + this.typeToCreate = typeToCreate; this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -110,7 +110,7 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(tp, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.typeToCreate, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -149,27 +149,16 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) - { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); - } - - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (ConstructorBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ConstructorBinding)GetManagedObject(ob); - int res = PyVisit(self.pyTypeHndl, visit, arg); + int res = PyVisit(self.typeToCreate.Handle, visit, arg); if (res != 0) return res; res = PyVisit(self.repr, visit, arg); @@ -190,15 +179,15 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject - private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. + private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; private ConstructorInfo ctorInfo; private IntPtr repr; - public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) + public BoundContructor(Type type, PyType typeToCreate, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - this.pyTypeHndl = pyTypeHndl; // steal a type reference + this.typeToCreate = typeToCreate; this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -229,7 +218,7 @@ public static IntPtr tp_call(IntPtr op, IntPtr args, IntPtr kw) } // Instantiate the python object that wraps the result of the method call // and return the PyObject* to it. - return CLRObject.GetInstHandle(obj, self.pyTypeHndl); + return CLRObject.GetInstHandle(obj, self.typeToCreate.Reference).DangerousMoveToPointer(); } /// @@ -252,27 +241,16 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } - /// - /// ConstructorBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) - { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.XDecref(self.repr); - self.Dealloc(); - } - - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (BoundContructor)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.repr); - return 0; + Runtime.Py_CLEAR(ref this.repr); + base.Clear(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (BoundContructor)GetManagedObject(ob); - int res = PyVisit(self.pyTypeHndl, visit, arg); + int res = PyVisit(self.typeToCreate.Handle, visit, arg); if (res != 0) return res; res = PyVisit(self.repr, visit, arg); diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index e1b689cf3..7cee0890c 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -27,7 +27,6 @@ private Converter() private static Type int16Type; private static Type int32Type; private static Type int64Type; - private static Type flagsType; private static Type boolType; private static Type typeType; @@ -42,7 +41,6 @@ static Converter() singleType = typeof(Single); doubleType = typeof(Double); decimalType = typeof(Decimal); - flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); } @@ -112,6 +110,9 @@ internal static IntPtr ToPython(T value) return ToPython(value, typeof(T)); } + internal static NewReference ToPythonReference(T value) + => NewReference.DangerousFromPointer(ToPython(value, typeof(T))); + private static readonly Func IsTransparentProxy = GetIsTransparentProxy(); private static bool Never(object _) => false; @@ -148,7 +149,11 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + if (Type.GetTypeCode(type) == TypeCode.Object + && value.GetType() != typeof(object) + && value is not Type + || type.IsEnum + ) { var encoded = PyObjectConversions.TryEncode(value, type); if (encoded != null) { result = encoded.Handle; @@ -197,12 +202,27 @@ internal static IntPtr ToPython(object value, Type type) return ClassDerivedObject.ToPython(pyderived); } + // ModuleObjects are created in a way that their wrapping them as + // a CLRObject fails, the ClassObject has no tpHandle. Return the + // pyHandle as is, do not convert. + if (value is ModuleObject modobj) + { + var handle = modobj.pyHandle; + Runtime.XIncref(handle); + return handle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. type = value.GetType(); + if (type.IsEnum) + { + return CLRObject.GetInstHandle(value, type); + } + TypeCode tc = Type.GetTypeCode(type); switch (tc) @@ -211,7 +231,7 @@ internal static IntPtr ToPython(object value, Type type) return CLRObject.GetInstHandle(value, type); case TypeCode.String: - return Runtime.PyUnicode_FromString((string)value); + return Runtime.PyString_FromString((string)value); case TypeCode.Int32: return Runtime.PyInt_FromInt32((int)value); @@ -241,9 +261,9 @@ internal static IntPtr ToPython(object value, Type type) // return Runtime.PyFloat_FromDouble((double)((float)value)); string ss = ((float)value).ToString(nfi); IntPtr ps = Runtime.PyString_FromString(ss); - IntPtr op = Runtime.PyFloat_FromString(ps, IntPtr.Zero); + NewReference op = Runtime.PyFloat_FromString(new BorrowedReference(ps));; Runtime.XDecref(ps); - return op; + return op.DangerousMoveToPointerOrNull(); case TypeCode.Double: return Runtime.PyFloat_FromDouble((double)value); @@ -303,6 +323,11 @@ internal static IntPtr ToPythonImplicit(object value) /// Return a managed object for the given Python object, taking funny /// byref types into account. /// + /// A Python object + /// The desired managed type + /// Receives the managed object + /// If true, call Exceptions.SetError with the reason for failure. + /// True on success internal static bool ToManaged(IntPtr value, Type type, out object result, bool setError) { @@ -312,6 +337,18 @@ internal static bool ToManaged(IntPtr value, Type type, } return Converter.ToManagedValue(value, type, out result, setError); } + /// + /// Return a managed object for the given Python object, taking funny + /// byref types into account. + /// + /// A Python object + /// The desired managed type + /// Receives the managed object + /// If true, call Exceptions.SetError with the reason for failure. + /// True on success + internal static bool ToManaged(BorrowedReference value, Type type, + out object result, bool setError) + => ToManaged(value.DangerousGetAddress(), type, out result, setError); internal static bool ToManagedValue(BorrowedReference value, Type obType, out object result, bool setError) @@ -333,20 +370,23 @@ internal static bool ToManagedValue(IntPtr value, Type obType, if (mt != null) { - if (mt is CLRObject) + if (mt is CLRObject co) { - object tmp = ((CLRObject)mt).inst; + object tmp = co.inst; if (obType.IsInstanceOfType(tmp)) { result = tmp; return true; } - Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); + if (setError) + { + string typeString = tmp is null ? "null" : tmp.GetType().ToString(); + Exceptions.SetError(Exceptions.TypeError, $"{typeString} value cannot be converted to {obType}"); + } return false; } - if (mt is ClassBase) + if (mt is ClassBase cb) { - var cb = (ClassBase)mt; if (!cb.type.Valid) { Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); @@ -376,14 +416,18 @@ internal static bool ToManagedValue(IntPtr value, Type obType, obType = obType.GetGenericArguments()[0]; } - if (obType.IsArray) + if (obType.ContainsGenericParameters) { - return ToArray(value, obType, out result, setError); + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, $"Cannot create an instance of the open generic type {obType}"); + } + return false; } - if (obType.IsEnum) + if (obType.IsArray) { - return ToEnum(value, obType, out result, setError); + return ToArray(value, obType, out result, setError); } // Conversion to 'Object' is done based on some reasonable default @@ -480,7 +524,7 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } TypeCode typeCode = Type.GetTypeCode(obType); - if (typeCode == TypeCode.Object) + if (typeCode == TypeCode.Object || obType.IsEnum) { IntPtr pyType = Runtime.PyObject_TYPE(value); if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) @@ -494,13 +538,32 @@ internal static bool ToManagedValue(IntPtr value, Type obType, internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + internal static int ToInt32(BorrowedReference value) + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + throw PythonException.ThrowLastAsClrException(); + } + return checked((int)num); + } + /// /// Convert a Python value to an instance of a primitive managed type. /// private static bool ToPrimitive(IntPtr value, Type obType, out object result, bool setError) { - TypeCode tc = Type.GetTypeCode(obType); result = null; + if (obType.IsEnum) + { + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, "since Python.NET 3.0 int can not be converted to Enum implicitly. Use Enum(int_value)"); + } + return false; + } + + TypeCode tc = Type.GetTypeCode(obType); IntPtr op = IntPtr.Zero; switch (tc) @@ -517,7 +580,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int32: { // Python3 always use PyLong API - long num = Runtime.PyLong_AsLongLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -540,14 +603,14 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -566,14 +629,14 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -592,7 +655,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } @@ -610,7 +673,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -625,7 +688,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int16: { - int num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -640,18 +703,35 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.Int64: { - long num = (long)Runtime.PyLong_AsLongLong(value); - if (num == -1 && Exceptions.ErrorOccurred()) + if (Runtime.Is32Bit) { - goto convert_error; + if (!Runtime.PyLong_Check(value)) + { + goto type_error; + } + long num = Runtime.PyExplicitlyConvertToInt64(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = num; + return true; + } + else + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = (long)num; + return true; } - result = num; - return true; } case TypeCode.UInt16: { - long num = Runtime.PyLong_AsLong(value); + nint num = Runtime.PyLong_AsSignedSize_t(value); if (num == -1 && Exceptions.ErrorOccurred()) { goto convert_error; @@ -666,61 +746,25 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo case TypeCode.UInt32: { - op = value; - if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) - { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - goto convert_error; - } - } - if (Runtime.Is32Bit || Runtime.IsWindows) + nuint num = Runtime.PyLong_AsUnsignedSize_t(value); + if (num == unchecked((nuint)(-1)) && Exceptions.ErrorOccurred()) { - uint num = Runtime.PyLong_AsUnsignedLong32(op); - if (num == uint.MaxValue && Exceptions.ErrorOccurred()) - { - goto convert_error; - } - result = num; + goto convert_error; } - else + if (num > UInt32.MaxValue) { - ulong num = Runtime.PyLong_AsUnsignedLong64(op); - if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) - { - goto convert_error; - } - try - { - result = Convert.ToUInt32(num); - } - catch (OverflowException) - { - // Probably wasn't an overflow in python but was in C# (e.g. if cpython - // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in - // PyLong_AsUnsignedLong) - goto overflow; - } + goto overflow; } + result = (uint)num; return true; } case TypeCode.UInt64: { - op = value; - if (Runtime.PyObject_TYPE(value) != Runtime.PyLongType) - { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - goto convert_error; - } - } - ulong num = Runtime.PyLong_AsUnsignedLongLong(op); + ulong num = Runtime.PyLong_AsUnsignedLongLong(value); if (num == ulong.MaxValue && Exceptions.ErrorOccurred()) { - goto overflow; + goto convert_error; } result = num; return true; @@ -793,10 +837,15 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo private static void SetConversionError(IntPtr value, Type target) { + // PyObject_Repr might clear the error + Runtime.PyErr_Fetch(out var causeType, out var causeVal, out var causeTrace); + IntPtr ob = Runtime.PyObject_Repr(value); string src = Runtime.GetManagedString(ob); Runtime.XDecref(ob); - Exceptions.SetError(Exceptions.TypeError, $"Cannot convert {src} to {target}"); + + Runtime.PyErr_Restore(causeType.StealNullable(), causeVal.StealNullable(), causeTrace.StealNullable()); + Exceptions.RaiseTypeError($"Cannot convert {src} to {target}"); } @@ -810,34 +859,61 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s Type elementType = obType.GetElementType(); result = null; - bool IsSeqObj = Runtime.PySequence_Check(value); - var len = IsSeqObj ? Runtime.PySequence_Size(value) : -1; - IntPtr IterObject = Runtime.PyObject_GetIter(value); - - if(IterObject==IntPtr.Zero) { + if (IterObject == IntPtr.Zero) + { if (setError) { SetConversionError(value, obType); } + else + { + // PyObject_GetIter will have set an error + Exceptions.Clear(); + } return false; } - Array items; + IList list; + try + { + // MakeGenericType can throw because elementType may not be a valid generic argument even though elementType[] is a valid array type. + // For example, if elementType is a pointer type. + // See https://docs.microsoft.com/en-us/dotnet/api/system.type.makegenerictype#System_Type_MakeGenericType_System_Type + var constructedListType = typeof(List<>).MakeGenericType(elementType); + bool IsSeqObj = Runtime.PySequence_Check(value); + if (IsSeqObj) + { + var len = Runtime.PySequence_Size(value); + list = (IList)Activator.CreateInstance(constructedListType, new Object[] { (int)len }); + } + else + { + // CreateInstance can throw even if MakeGenericType succeeded. + // See https://docs.microsoft.com/en-us/dotnet/api/system.activator.createinstance#System_Activator_CreateInstance_System_Type_ + list = (IList)Activator.CreateInstance(constructedListType); + } + } + catch (Exception e) + { + if (setError) + { + Exceptions.SetError(e); + SetConversionError(value, obType); + } + return false; + } - var listType = typeof(List<>); - var constructedListType = listType.MakeGenericType(elementType); - IList list = IsSeqObj ? (IList) Activator.CreateInstance(constructedListType, new Object[] {(int) len}) : - (IList) Activator.CreateInstance(constructedListType); IntPtr item; while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) { - object obj = null; + object obj; - if (!Converter.ToManaged(item, elementType, out obj, true)) + if (!Converter.ToManaged(item, elementType, out obj, setError)) { Runtime.XDecref(item); + Runtime.XDecref(IterObject); return false; } @@ -846,45 +922,17 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s } Runtime.XDecref(IterObject); - items = Array.CreateInstance(elementType, list.Count); - list.CopyTo(items, 0); - - result = items; - return true; - } - - - /// - /// Convert a Python value to a correctly typed managed enum instance. - /// - private static bool ToEnum(IntPtr value, Type obType, out object result, bool setError) - { - Type etype = Enum.GetUnderlyingType(obType); - result = null; - - if (!ToPrimitive(value, etype, out result, setError)) + if (Exceptions.ErrorOccurred()) { + if (!setError) Exceptions.Clear(); return false; } - if (Enum.IsDefined(obType, result)) - { - result = Enum.ToObject(obType, result); - return true; - } - - if (obType.GetCustomAttributes(flagsType, true).Length > 0) - { - result = Enum.ToObject(obType, result); - return true; - } - - if (setError) - { - Exceptions.SetError(Exceptions.ValueError, "invalid enumeration value"); - } + Array items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); - return false; + result = items; + return true; } } diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs index b10d0c59f..5711b9f87 100644 --- a/src/runtime/converterextensions.cs +++ b/src/runtime/converterextensions.cs @@ -107,6 +107,8 @@ static IPyObjectEncoder[] GetEncoders(Type type) #region Decoding static readonly ConcurrentDictionary pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(BorrowedReference value, BorrowedReference type, Type targetType, out object result) + => TryDecode(value.DangerousGetAddress(), type.DangerousGetAddress(), targetType, out result); internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) { if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 3e6541c44..22f603400 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -1,7 +1,9 @@ using System; -using System.Collections; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Reflection.Emit; +using System.Text; namespace Python.Runtime { @@ -11,23 +13,20 @@ namespace Python.Runtime /// internal class DelegateManager { - private Hashtable cache; - private Type basetype; - private Type listtype; - private Type voidtype; - private Type typetype; - private Type ptrtype; - private CodeGenerator codeGenerator; + private readonly Dictionary cache = new Dictionary(); + private readonly Type basetype = typeof(Dispatcher); + private readonly Type arrayType = typeof(object[]); + private readonly Type voidtype = typeof(void); + private readonly Type typetype = typeof(Type); + private readonly Type ptrtype = typeof(IntPtr); + private readonly CodeGenerator codeGenerator = new CodeGenerator(); + private readonly ConstructorInfo arrayCtor; + private readonly MethodInfo dispatch; public DelegateManager() { - basetype = typeof(Dispatcher); - listtype = typeof(ArrayList); - voidtype = typeof(void); - typetype = typeof(Type); - ptrtype = typeof(IntPtr); - cache = new Hashtable(); - codeGenerator = new CodeGenerator(); + arrayCtor = arrayType.GetConstructor(new[] { typeof(int) }); + dispatch = basetype.GetMethod("Dispatch"); } /// @@ -58,10 +57,9 @@ private Type GetDispatcher(Type dtype) // unique signatures rather than delegate types, since multiple // delegate types with the same sig could use the same dispatcher. - object item = cache[dtype]; - if (item != null) + if (cache.TryGetValue(dtype, out Type item)) { - return (Type)item; + return item; } string name = $"__{dtype.FullName}Dispatcher"; @@ -103,34 +101,77 @@ private Type GetDispatcher(Type dtype) MethodBuilder mb = tb.DefineMethod("Invoke", MethodAttributes.Public, method.ReturnType, signature); - ConstructorInfo ctor = listtype.GetConstructor(Type.EmptyTypes); - MethodInfo dispatch = basetype.GetMethod("Dispatch"); - MethodInfo add = listtype.GetMethod("Add"); - il = mb.GetILGenerator(); - il.DeclareLocal(listtype); - il.Emit(OpCodes.Newobj, ctor); + // loc_0 = new object[pi.Length] + il.DeclareLocal(arrayType); + il.Emit(OpCodes.Ldc_I4, pi.Length); + il.Emit(OpCodes.Newobj, arrayCtor); il.Emit(OpCodes.Stloc_0); + bool anyByRef = false; + for (var c = 0; c < signature.Length; c++) { Type t = signature[c]; il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + if (t.IsByRef) + { + // The argument is a pointer. We must dereference the pointer to get the value or object it points to. + t = t.GetElementType(); + if (t.IsValueType) + { + il.Emit(OpCodes.Ldobj, t); + } + else + { + il.Emit(OpCodes.Ldind_Ref); + } + anyByRef = true; + } + if (t.IsValueType) { il.Emit(OpCodes.Box, t); } - il.Emit(OpCodes.Callvirt, add); - il.Emit(OpCodes.Pop); + // args[c] = arg + il.Emit(OpCodes.Stelem_Ref); } il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Call, dispatch); + if (anyByRef) + { + // Dispatch() will have modified elements of the args list that correspond to out parameters. + for (var c = 0; c < signature.Length; c++) + { + Type t = signature[c]; + if (t.IsByRef) + { + t = t.GetElementType(); + // *arg = args[c] + il.Emit(OpCodes.Ldarg_S, (byte)(c + 1)); + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, c); + il.Emit(OpCodes.Ldelem_Ref); + if (t.IsValueType) + { + il.Emit(OpCodes.Unbox_Any, t); + il.Emit(OpCodes.Stobj, t); + } + else + { + il.Emit(OpCodes.Stind_Ref); + } + } + } + } + if (method.ReturnType == voidtype) { il.Emit(OpCodes.Pop); @@ -218,7 +259,7 @@ public void Dispose() GC.SuppressFinalize(this); } - public object Dispatch(ArrayList args) + public object Dispatch(object[] args) { IntPtr gs = PythonEngine.AcquireLock(); object ob; @@ -235,7 +276,7 @@ public object Dispatch(ArrayList args) return ob; } - public object TrueDispatch(ArrayList args) + private object TrueDispatch(object[] args) { MethodInfo method = dtype.GetMethod("Invoke"); ParameterInfo[] pi = method.GetParameters(); @@ -255,24 +296,111 @@ public object TrueDispatch(ArrayList args) if (op == IntPtr.Zero) { - var e = new PythonException(); - throw e; + throw PythonException.ThrowLastAsClrException(); } - if (rtype == typeof(void)) + try { - return null; - } + int byRefCount = pi.Count(parameterInfo => parameterInfo.ParameterType.IsByRef); + if (byRefCount > 0) + { + // By symmetry with MethodBinder.Invoke, when there are out + // parameters we expect to receive a tuple containing + // the result, if any, followed by the out parameters. If there is only + // one out parameter and the return type of the method is void, + // we instead receive the out parameter as the result from Python. + + bool isVoid = rtype == typeof(void); + int tupleSize = byRefCount + (isVoid ? 0 : 1); + if (isVoid && byRefCount == 1) + { + // The return type is void and there is a single out parameter. + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (!Converter.ToManaged(op, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function did not return {t.GetElementType()} (the out parameter type)"); + throw PythonException.ThrowLastAsClrException(); + } + args[i] = newArg; + break; + } + } + return null; + } + else if (Runtime.PyTuple_Check(op) && Runtime.PyTuple_Size(op) == tupleSize) + { + int index = isVoid ? 0 : 1; + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + IntPtr item = Runtime.PyTuple_GetItem(op, index++); + if (!Converter.ToManaged(item, t, out object newArg, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element {i} was not {t.GetElementType()} (the out parameter type)"); + throw PythonException.ThrowLastAsClrException(); + } + args[i] = newArg; + } + } + if (isVoid) + { + return null; + } + IntPtr item0 = Runtime.PyTuple_GetItem(op, 0); + if (!Converter.ToManaged(item0, rtype, out object result0, true)) + { + Exceptions.RaiseTypeError($"The Python function returned a tuple where element 0 was not {rtype} (the return type)"); + throw PythonException.ThrowLastAsClrException(); + } + return result0; + } + else + { + string tpName = Runtime.PyObject_GetTypeName(op); + if (Runtime.PyTuple_Check(op)) + { + tpName += $" of size {Runtime.PyTuple_Size(op)}"; + } + StringBuilder sb = new StringBuilder(); + if (!isVoid) sb.Append(rtype.FullName); + for (int i = 0; i < pi.Length; i++) + { + Type t = pi[i].ParameterType; + if (t.IsByRef) + { + if (sb.Length > 0) sb.Append(","); + sb.Append(t.GetElementType().FullName); + } + } + string returnValueString = isVoid ? "" : "the return value and "; + Exceptions.RaiseTypeError($"Expected a tuple ({sb}) of {returnValueString}the values for out and ref parameters, got {tpName}."); + throw PythonException.ThrowLastAsClrException(); + } + } + + if (rtype == typeof(void)) + { + return null; + } - object result; - if (!Converter.ToManaged(op, rtype, out result, true)) + object result; + if (!Converter.ToManaged(op, rtype, out result, true)) + { + throw PythonException.ThrowLastAsClrException(); + } + + return result; + } + finally { Runtime.XDecref(op); - throw new PythonException(); } - - Runtime.XDecref(op); - return result; } } } diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 581095185..65c8fdccf 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -68,35 +68,27 @@ public static IntPtr nb_inplace_subtract(IntPtr ob, IntPtr arg) /// /// EventBinding __hash__ implementation. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); - long x = 0; - long y = 0; + nint x = 0; if (self.target != IntPtr.Zero) { - x = Runtime.PyObject_Hash(self.target).ToInt64(); + x = Runtime.PyObject_Hash(self.target); if (x == -1) { - return new IntPtr(-1); + return x; } } - y = Runtime.PyObject_Hash(self.e.pyHandle).ToInt64(); + nint y = Runtime.PyObject_Hash(self.e.pyHandle); if (y == -1) { - return new IntPtr(-1); + return y; } - x ^= y; - - if (x == -1) - { - x = -1; - } - - return new IntPtr(x); + return x ^ y; } @@ -111,22 +103,10 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString(s); } - - /// - /// EventBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) - { - var self = (EventBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); - } - - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (EventBinding)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.target); - return 0; + Runtime.Py_CLEAR(ref this.target); + base.Clear(); } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 0f2796a14..941bbdf46 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -72,6 +72,12 @@ internal bool AddEventHandler(IntPtr target, IntPtr handler) /// internal bool RemoveEventHandler(IntPtr target, IntPtr handler) { + if (reg == null) + { + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + object obj = null; if (target != IntPtr.Zero) { @@ -79,10 +85,9 @@ internal bool RemoveEventHandler(IntPtr target, IntPtr handler) obj = co.inst; } - IntPtr hash = Runtime.PyObject_Hash(handler); - if (Exceptions.ErrorOccurred() || reg == null) + nint hash = Runtime.PyObject_Hash(handler); + if (hash == -1 && Exceptions.ErrorOccurred()) { - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); return false; } @@ -193,17 +198,14 @@ public static IntPtr tp_repr(IntPtr ob) } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Clear() { - var self = (EventObject)GetManagedObject(ob); - if (self.unbound != null) + if (this.unbound is not null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; } - self.Dealloc(); + base.Clear(); } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index ab28905d2..a612e34e3 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -1,5 +1,7 @@ using System; +using System.Diagnostics; using System.Reflection; +using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; namespace Python.Runtime @@ -22,19 +24,10 @@ internal ExceptionClassObject(Type tp) : base(tp) { } - internal static Exception ToException(IntPtr ob) + internal static Exception ToException(BorrowedReference ob) { var co = GetManagedObject(ob) as CLRObject; - if (co == null) - { - return null; - } - var e = co.inst as Exception; - if (e == null) - { - return null; - } - return e; + return co?.inst as Exception; } /// @@ -42,7 +35,7 @@ internal static Exception ToException(IntPtr ob) /// public new static IntPtr tp_repr(IntPtr ob) { - Exception e = ToException(ob); + Exception e = ToException(new BorrowedReference(ob)); if (e == null) { return Exceptions.RaiseTypeError("invalid object"); @@ -57,7 +50,7 @@ internal static Exception ToException(IntPtr ob) { message = String.Format("{0}()", name); } - return Runtime.PyUnicode_FromString(message); + return Runtime.PyString_FromString(message); } /// @@ -65,22 +58,24 @@ internal static Exception ToException(IntPtr ob) /// public new static IntPtr tp_str(IntPtr ob) { - Exception e = ToException(ob); + Exception e = ToException(new BorrowedReference(ob)); if (e == null) { return Exceptions.RaiseTypeError("invalid object"); } - string message = string.Empty; - if (e.Message != string.Empty) + string message = e.ToString(); + string fullTypeName = e.GetType().FullName; + string prefix = fullTypeName + ": "; + if (message.StartsWith(prefix)) { - message = e.Message; + message = message.Substring(prefix.Length); } - if (!string.IsNullOrEmpty(e.StackTrace)) + else if (message.StartsWith(fullTypeName)) { - message = message + "\n" + e.StackTrace; + message = message.Substring(fullTypeName.Length); } - return Runtime.PyUnicode_FromString(message); + return Runtime.PyString_FromString(message); } } @@ -90,10 +85,10 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public static class Exceptions + internal static class Exceptions { - internal static IntPtr warnings_module; - internal static IntPtr exceptions_module; + internal static PyModule warnings_module; + internal static PyModule exceptions_module; /// /// Initialization performed on startup of the Python runtime. @@ -101,15 +96,12 @@ public static class Exceptions internal static void Initialize() { string exceptionsModuleName = "builtins"; - exceptions_module = Runtime.PyImport_ImportModule(exceptionsModuleName); - - Exceptions.ErrorCheck(exceptions_module); - warnings_module = Runtime.PyImport_ImportModule("warnings"); - Exceptions.ErrorCheck(warnings_module); + exceptions_module = PyModule.Import(exceptionsModuleName); + warnings_module = PyModule.Import("warnings"); Type type = typeof(Exceptions); foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { - IntPtr op = Runtime.PyObject_GetAttrString(exceptions_module, fi.Name); + IntPtr op = Runtime.PyObject_GetAttrString(exceptions_module.obj, fi.Name); if (op != IntPtr.Zero) { fi.SetValue(type, op); @@ -144,8 +136,8 @@ internal static void Shutdown() Runtime.XDecref(op); fi.SetValue(null, IntPtr.Zero); } - Runtime.Py_CLEAR(ref exceptions_module); - Runtime.Py_CLEAR(ref warnings_module); + exceptions_module.Dispose(); + warnings_module.Dispose(); } /// @@ -155,21 +147,13 @@ internal static void Shutdown() /// __getattr__ implementation, and thus dereferencing a NULL /// pointer. /// - /// The python object wrapping - internal static void SetArgsAndCause(IntPtr ob) + internal static void SetArgsAndCause(BorrowedReference ob, Exception e) { - // e: A CLR Exception - Exception e = ExceptionClassObject.ToException(ob); - if (e == null) - { - return; - } - IntPtr args; if (!string.IsNullOrEmpty(e.Message)) { args = Runtime.PyTuple_New(1); - IntPtr msg = Runtime.PyUnicode_FromString(e.Message); + IntPtr msg = Runtime.PyString_FromString(e.Message); Runtime.PyTuple_SetItem(args, 0, msg); } else @@ -177,12 +161,16 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); + using var argsTuple = NewReference.DangerousFromPointer(args); + + if (Runtime.PyObject_SetAttrString(ob, "args", argsTuple) != 0) + throw PythonException.ThrowLastAsClrException(); if (e.InnerException != null) { - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. + using var cause = CLRObject.GetReference(e.InnerException); + Runtime.PyException_SetCause(ob, cause.Steal()); } } @@ -190,14 +178,16 @@ internal static void SetArgsAndCause(IntPtr ob) /// Shortcut for (pointer == NULL) -> throw PythonException /// /// Pointer to a Python object - internal static void ErrorCheck(IntPtr pointer) + internal static void ErrorCheck(BorrowedReference pointer) { - if (pointer == IntPtr.Zero) + if (pointer.IsNull) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } + internal static void ErrorCheck(IntPtr pointer) => ErrorCheck(new BorrowedReference(pointer)); + /// /// Shortcut for (pointer == NULL or ErrorOccurred()) -> throw PythonException /// @@ -205,10 +195,19 @@ internal static void ErrorOccurredCheck(IntPtr pointer) { if (pointer == IntPtr.Zero || ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } + internal static IntPtr ErrorCheckIfNull(IntPtr pointer) + { + if (pointer == IntPtr.Zero && ErrorOccurred()) + { + throw PythonException.ThrowLastAsClrException(); + } + return pointer; + } + /// /// ExceptionMatches Method /// @@ -221,19 +220,6 @@ public static bool ExceptionMatches(IntPtr ob) return Runtime.PyErr_ExceptionMatches(ob) != 0; } - /// - /// ExceptionMatches Method - /// - /// - /// Returns true if the given Python exception matches the given - /// Python object. This is a wrapper for PyErr_GivenExceptionMatches. - /// - public static bool ExceptionMatches(IntPtr exc, IntPtr ob) - { - int i = Runtime.PyErr_GivenExceptionMatches(exc, ob); - return i != 0; - } - /// /// SetError Method /// @@ -258,6 +244,7 @@ public static void SetError(IntPtr type, IntPtr exceptionObject) Runtime.PyErr_SetObject(new BorrowedReference(type), new BorrowedReference(exceptionObject)); } + internal const string DispatchInfoAttribute = "__dispatch_info__"; /// /// SetError Method /// @@ -266,8 +253,10 @@ public static void SetError(IntPtr type, IntPtr exceptionObject) /// object. The CLR exception instance is wrapped as a Python /// object, allowing it to be handled naturally from Python. /// - public static void SetError(Exception e) + public static bool SetError(Exception e) { + Debug.Assert(e is not null); + // Because delegates allow arbitrary nesting of Python calling // managed calling Python calling... etc. it is possible that we // might get a managed exception raised that is a wrapper for a @@ -276,18 +265,37 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); - return; + pe.Restore(); + return true; } - IntPtr op = CLRObject.GetInstHandle(e); - IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__); - Runtime.PyErr_SetObject(new BorrowedReference(etype), new BorrowedReference(op)); - Runtime.XDecref(etype); - Runtime.XDecref(op); + using var instance = Converter.ToPythonReference(e); + if (instance.IsNull()) return false; + + var exceptionInfo = ExceptionDispatchInfo.Capture(e); + using var pyInfo = Converter.ToPythonReference(exceptionInfo); + + if (Runtime.PyObject_SetAttrString(instance, DispatchInfoAttribute, pyInfo) != 0) + return false; + + Debug.Assert(Runtime.PyObject_TypeCheck(instance, new BorrowedReference(BaseException))); + + var type = Runtime.PyObject_TYPE(instance); + Runtime.PyErr_SetObject(type, instance); + return true; + } + + /// + /// When called after SetError, sets the cause of the error. + /// + /// The cause of the current error + public static void SetCause(Exception cause) + { + var currentException = PythonException.FetchCurrentRaw(); + currentException.Normalize(); + using var causeInstance = Converter.ToPythonReference(cause); + Runtime.PyException_SetCause(currentException.Value!.Reference, causeInstance.Steal()); + currentException.Restore(); } /// @@ -299,7 +307,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != null; } /// @@ -323,14 +331,12 @@ public static void Clear() public static void warn(string message, IntPtr exception, int stacklevel) { if (exception == IntPtr.Zero || - (Runtime.PyObject_IsSubclass(exception, Exceptions.Warning) != 1)) + (Runtime.PyObject_IsSubclass(new BorrowedReference(exception), new BorrowedReference(Exceptions.Warning)) != 1)) { Exceptions.RaiseTypeError("Invalid exception"); } - Runtime.XIncref(warnings_module); - IntPtr warn = Runtime.PyObject_GetAttrString(warnings_module, "warn"); - Runtime.XDecref(warnings_module); + IntPtr warn = Runtime.PyObject_GetAttrString(warnings_module.obj, "warn"); Exceptions.ErrorCheck(warn); IntPtr args = Runtime.PyTuple_New(3); @@ -368,17 +374,36 @@ public static void deprecation(string message) // Internal helper methods for common error handling scenarios. //==================================================================== + /// + /// Raises a TypeError exception and attaches any existing exception as its cause. + /// + /// The exception message + /// IntPtr.Zero internal static IntPtr RaiseTypeError(string message) { + var cause = PythonException.FetchCurrentOrNullRaw(); + cause?.Normalize(); + Exceptions.SetError(Exceptions.TypeError, message); + + if (cause is null) return IntPtr.Zero; + + var typeError = PythonException.FetchCurrentRaw(); + typeError.Normalize(); + + Runtime.PyException_SetCause( + typeError.Value!.Reference, + new NewReference(cause.Value!.Reference).Steal()); + typeError.Restore(); + return IntPtr.Zero; } // 2010-11-16: Arranged in python (2.6 & 2.7) source header file order /* Predefined exceptions are - puplic static variables on the Exceptions class filled in from + public static variables on the Exceptions class filled in from the python class using reflection in Initialize() looked up by - name, not posistion. */ + name, not position. */ public static IntPtr BaseException; public static IntPtr Exception; public static IntPtr StopIteration; diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index a5f0f1219..0ebd7ec4c 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -18,7 +18,7 @@ public ExtensionType() // The Python instance object is related to an instance of a // particular concrete subclass with a hidden CLR gchandle. - IntPtr tp = TypeManager.GetTypeHandle(GetType()); + BorrowedReference tp = TypeManager.GetTypeReference(GetType()); //int rc = (int)Marshal.ReadIntPtr(tp, TypeOffset.ob_refcnt); //if (rc > 1050) @@ -27,19 +27,23 @@ public ExtensionType() // DebugUtil.DumpType(tp); //} - IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); + NewReference py = Runtime.PyType_GenericAlloc(tp, 0); - // Steals a ref to tpHandle. - tpHandle = tp; - pyHandle = py; + // Borrowed reference. Valid as long as pyHandle is valid. + tpHandle = tp.DangerousGetAddress(); + pyHandle = py.DangerousMoveToPointer(); +#if DEBUG + GetGCHandle(ObjectReference, TypeReference, out var existing); + System.Diagnostics.Debug.Assert(existing == IntPtr.Zero); +#endif SetupGc(); } void SetupGc () { GCHandle gc = AllocGCHandle(TrackTypes.Extension); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + InitGCHandle(ObjectReference, TypeReference, gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most @@ -50,20 +54,23 @@ void SetupGc () } - /// - /// Common finalization code to support custom tp_deallocs. - /// - public static void FinalizeObject(ManagedType self) + protected virtual void Dealloc() { - ClearObjectDict(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); - // Not necessary for decref of `tpHandle`. - self.FreeGCHandle(); + var type = Runtime.PyObject_TYPE(this.ObjectReference); + Runtime.PyObject_GC_Del(this.pyHandle); + // Not necessary for decref of `tpHandle` - it is borrowed + + this.FreeGCHandle(); + + // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc + Runtime.XDecref(type.DangerousGetAddress()); } - protected void Dealloc() + /// DecRefs and nulls any fields pointing back to Python + protected virtual void Clear() { - FinalizeObject(this); + ClearObjectDict(this.pyHandle); + // Not necessary for decref of `tpHandle` - it is borrowed } /// @@ -76,7 +83,7 @@ public static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) { message = "readonly attribute"; } - Exceptions.SetError(Exceptions.TypeError, message); + Exceptions.SetError(Exceptions.AttributeError, message); return -1; } @@ -92,15 +99,20 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } - /// - /// Default dealloc implementation. - /// public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. var self = (ExtensionType)GetManagedObject(ob); - self.Dealloc(); + self?.Clear(); + self?.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (ExtensionType)GetManagedObject(ob); + self?.Clear(); + return 0; } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index fe2e46aac..cfff54070 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -54,7 +54,7 @@ public class IncorrectRefCountException : Exception public IncorrectRefCountException(IntPtr ptr) { PyPtr = ptr; - IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + IntPtr pyname = Runtime.PyObject_Str(PyPtr); string name = Runtime.GetManagedString(pyname); Runtime.XDecref(pyname); _message = $"<{name}> may has a incorrect ref count"; @@ -160,7 +160,7 @@ private void DisposeAll() { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType, errVal, traceback); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index af6174188..1111adc28 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -9,94 +9,63 @@ namespace Python.Runtime /// internal static class ImportHook { - private static IntPtr py_import; private static CLRModule root; - private static MethodWrapper hook; private static IntPtr py_clr_module; - - private static IntPtr module_def = IntPtr.Zero; - - internal static void InitializeModuleDef() - { - if (module_def == IntPtr.Zero) - { - module_def = ModuleDefOffset.AllocModuleDef("clr"); - } - } - - internal static void ReleaseModuleDef() - { - if (module_def == IntPtr.Zero) - { - return; - } - ModuleDefOffset.FreeModuleDef(module_def); - module_def = IntPtr.Zero; - } - - /// - /// Initialize just the __import__ hook itself. - /// - static void InitImport() - { - // We replace the built-in Python __import__ with our own: first - // look in CLR modules, then if we don't find any call the default - // Python __import__. - IntPtr builtins = Runtime.GetBuiltins(); - py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - PythonException.ThrowIfIsNull(py_import); - - hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr); - PythonException.ThrowIfIsNotZero(res); - - Runtime.XDecref(builtins); - } - - /// - /// Restore the __import__ hook. - /// - static void RestoreImport() - { - IntPtr builtins = Runtime.GetBuiltins(); - - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); - PythonException.ThrowIfIsNotZero(res); - Runtime.XDecref(py_import); - py_import = IntPtr.Zero; - - hook.Release(); - hook = null; - - Runtime.XDecref(builtins); - } + static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); + + private const string LoaderCode = @" +import importlib.abc +import sys + +class DotNetLoader(importlib.abc.Loader): + + @classmethod + def exec_module(klass, mod): + # This method needs to exist. + pass + + @classmethod + def create_module(klass, spec): + import clr + return clr._load_clr_module(spec) + +class DotNetFinder(importlib.abc.MetaPathFinder): + + @classmethod + def find_spec(klass, fullname, paths=None, target=None): + # Don't import, we might call ourselves recursively! + if 'clr' not in sys.modules: + return None + clr = sys.modules['clr'] + if clr._available_namespaces and fullname in clr._available_namespaces: + return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) + return None + "; + const string availableNsKey = "_available_namespaces"; /// /// Initialization performed on startup of the Python runtime. /// - internal static void Initialize() + internal static unsafe void Initialize() { - InitImport(); - // Initialize the clr module and tell Python about it. root = new CLRModule(); // create a python module with the same methods as the clr module-like object - InitializeModuleDef(); - py_clr_module = Runtime.PyModule_Create2(module_def, 3); + py_clr_module = Runtime.PyModule_New("clr").DangerousMoveToPointer(); // both dicts are borrowed references - IntPtr mod_dict = Runtime.PyModule_GetDict(py_clr_module); - IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** - clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); Runtime.PyDict_Update(mod_dict, clr_dict); - IntPtr dict = Runtime.PyImport_GetModuleDict(); - Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); - Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); + SetupImportHook(); } - /// /// Cleanup resources upon shutdown of the Python runtime. /// @@ -107,15 +76,9 @@ internal static void Shutdown() return; } - RestoreImport(); - - bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + TeardownNameSpaceTracking(); Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; - if (shouldFreeDef) - { - ReleaseModuleDef(); - } Runtime.XDecref(root.pyHandle); root = null; @@ -134,199 +97,133 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) internal static void RestoreRuntimeData(RuntimeDataStorage storage) { - InitImport(); storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); } - /// - /// Return the clr python module (new reference) - /// - public static IntPtr GetCLRModule(IntPtr? fromList = null) + static void SetupImportHook() { - root.InitializePreload(); - - // update the module dictionary with the contents of the root dictionary - root.LoadNames(); - IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module); - IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** - clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); - Runtime.PyDict_Update(py_mod_dict, clr_dict); - - // find any items from the from list and get them from the root if they're not - // already in the module dictionary - if (fromList != null && fromList != IntPtr.Zero) + // Create the import hook module + var import_hook_module = Runtime.PyModule_New("clr.loader"); + + // Run the python code to create the module's classes. + var builtins = Runtime.PyEval_GetBuiltins(); + var exec = Runtime.PyDict_GetItemString(builtins, "exec"); + using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); + + var codeStr = NewReference.DangerousFromPointer(Runtime.PyString_FromString(LoaderCode)); + Runtime.PyTuple_SetItem(args, 0, codeStr); + var mod_dict = Runtime.PyModule_GetDict(import_hook_module); + // reference not stolen due to overload incref'ing for us. + Runtime.PyTuple_SetItem(args, 1, mod_dict); + Runtime.PyObject_Call(exec, args, default); + // Set as a sub-module of clr. + if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module.DangerousGetAddress()) != 0) { - if (Runtime.PyTuple_Check(fromList.GetValueOrDefault())) - { - Runtime.XIncref(py_mod_dict); - using (var mod_dict = new PyDict(py_mod_dict)) - { - Runtime.XIncref(fromList.GetValueOrDefault()); - using (var from = new PyTuple(fromList.GetValueOrDefault())) - { - foreach (PyObject item in from) - { - if (mod_dict.HasKey(item)) - { - continue; - } - - var s = item.AsManagedObject(typeof(string)) as string; - if (s == null) - { - continue; - } - - ManagedType attr = root.GetAttribute(s, true); - if (attr == null) - { - continue; - } - - Runtime.XIncref(attr.pyHandle); - using (var obj = new PyObject(attr.pyHandle)) - { - mod_dict.SetItem(s, obj); - } - } - } - } - } + Runtime.XDecref(import_hook_module.DangerousGetAddress()); + throw PythonException.ThrowLastAsClrException(); } - Runtime.XIncref(py_clr_module); - return py_clr_module; + + // Finally, add the hook to the meta path + var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder"); + var finderCtorArgs = NewReference.DangerousFromPointer(Runtime.PyTuple_New(0)); + var finder_inst = Runtime.PyObject_CallObject(findercls, finderCtorArgs); + var metapath = Runtime.PySys_GetObject("meta_path"); + Runtime.PyList_Append(metapath, finder_inst); } /// - /// The actual import hook that ties Python to the managed world. + /// Sets up the tracking of loaded namespaces. This makes available to + /// Python, as a Python object, the loaded namespaces. The set of loaded + /// namespaces is used during the import to verify if we can import a + /// CLR assembly as a module or not. The set is stored on the clr module. /// - public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) + static void SetupNamespaceTracking() { - // Replacement for the builtin __import__. The original import - // hook is saved as this.py_import. This version handles CLR - // import and defers to the normal builtin for everything else. - - var num_args = Runtime.PyTuple_Size(args); - if (num_args < 1) - { - return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); - } - - // borrowed reference - IntPtr py_mod_name = Runtime.PyTuple_GetItem(args, 0); - if (py_mod_name == IntPtr.Zero || - !Runtime.IsStringType(py_mod_name)) + using var newset = Runtime.PySet_New(default); + foreach (var ns in AssemblyManager.GetNamespaces()) { - return Exceptions.RaiseTypeError("string expected"); - } - - // Check whether the import is of the form 'from x import y'. - // This determines whether we return the head or tail module. - - IntPtr fromList = IntPtr.Zero; - var fromlist = false; - if (num_args >= 4) - { - fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != IntPtr.Zero && - Runtime.PyObject_IsTrue(fromList) == 1) + using var pyNs = NewReference.DangerousFromPointer(Runtime.PyString_FromString(ns)); + if (Runtime.PySet_Add(newset, pyNs) != 0) { - fromlist = true; + throw PythonException.ThrowLastAsClrException(); } - } - - string mod_name = Runtime.GetManagedString(py_mod_name); - // Check these BEFORE the built-in import runs; may as well - // do the Incref()ed return here, since we've already found - // the module. - if (mod_name == "clr") - { - IntPtr clr_module = GetCLRModule(fromList); - if (clr_module != IntPtr.Zero) + if (Runtime.PyDict_SetItemString(root.DictRef, availableNsKey, newset) != 0) { - IntPtr sys_modules = Runtime.PyImport_GetModuleDict(); - if (sys_modules != IntPtr.Zero) - { - Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); - } + throw PythonException.ThrowLastAsClrException(); } - return clr_module; } - string realname = mod_name; - string clr_prefix = null; - - // 2010-08-15: Always seemed smart to let python try first... - // This shaves off a few tenths of a second on test_module.py - // and works around a quirk where 'sys' is found by the - // LoadImplicit() deprecation logic. - // Turns out that the AssemblyManager.ResolveHandler() checks to see if any - // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very - // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args, kw); - if (res != IntPtr.Zero) + } + + /// + /// Removes the set of available namespaces from the clr module. + /// + static void TeardownNameSpaceTracking() + { + // If the C# runtime isn't loaded, then there are no namespaces available + Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); + } + + public static void AddNamespace(string name) + { + var pyNs = Runtime.PyString_FromString(name); + try { - // There was no error. - if (fromlist && IsLoadAll(fromList)) + var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); + if (!(nsSet.IsNull || nsSet.DangerousGetAddress() == Runtime.PyNone)) { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); + if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } } - return res; } - // There was an error - if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) + finally { - // and it was NOT an ImportError; bail out here. - return IntPtr.Zero; + Runtime.XDecref(pyNs); } + } - if (mod_name == string.Empty) - { - // Most likely a missing relative import. - // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: - // from . import _html5lib - // We don't support them anyway - return IntPtr.Zero; - } - // Save the exception - var originalException = new PythonException(); - // Otherwise, just clear the it. - Exceptions.Clear(); - string[] names = realname.Split('.'); + /// + /// Because we use a proxy module for the clr module, we somtimes need + /// to force the py_clr_module to sync with the actual clr module's dict. + /// + internal static void UpdateCLRModuleDict() + { + root.InitializePreload(); - // See if sys.modules for this interpreter already has the - // requested module. If so, just return the existing module. - IntPtr modules = Runtime.PyImport_GetModuleDict(); - IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name); + // update the module dictionary with the contents of the root dictionary + root.LoadNames(); + BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); - if (module != IntPtr.Zero) - { - if (fromlist) - { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } - Runtime.XIncref(module); - return module; - } - if (clr_prefix != null) - { - return GetCLRModule(fromList); - } - module = Runtime.PyDict_GetItemString(modules, names[0]); - Runtime.XIncref(module); - return module; - } - Exceptions.Clear(); + Runtime.PyDict_Update(py_mod_dict, clr_dict); + } + + /// + /// Return the clr python module (new reference) + /// + public static unsafe NewReference GetCLRModule() + { + UpdateCLRModuleDict(); + Runtime.XIncref(py_clr_module); + return NewReference.DangerousFromPointer(py_clr_module); + } - // Traverse the qualified module name to get the named module - // and place references in sys.modules as we go. Note that if + /// + /// The hook to import a CLR module into Python. Returns a new reference + /// to the module. + /// + public static ModuleObject Import(string modname) + { + // Traverse the qualified module name to get the named module. + // Note that if // we are running in interactive mode we pre-load the names in // each module, which is often useful for introspection. If we // are not interactive, we stick to just-in-time creation of @@ -335,17 +232,18 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // enable preloading in a non-interactive python processing by // setting clr.preload = True - ModuleObject head = mod_name == realname ? null : root; + ModuleObject head = null; ModuleObject tail = root; root.InitializePreload(); + string[] names = modname.Split('.'); foreach (string name in names) { ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - originalException.Restore(); - return IntPtr.Zero; + Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); + throw PythonException.ThrowLastAsClrException(); } if (head == null) { @@ -356,32 +254,15 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { tail.LoadNames(); } - - // Add the module to sys.modules - Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.pyHandle); - - // If imported from CLR add clr. to sys.modules as well - if (clr_prefix != null) - { - Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.pyHandle); - } - } - - { - var mod = fromlist ? tail : head; - - if (fromlist && IsLoadAll(fromList)) - { - mod.LoadNames(); - } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; } + tail.IncrRefCount(); + return tail; } - private static bool IsLoadAll(IntPtr fromList) + private static bool IsLoadAll(BorrowedReference fromList) { + if (fromList == null) throw new ArgumentNullException(nameof(fromList)); + if (CLRModule.preload) { return false; @@ -390,10 +271,8 @@ private static bool IsLoadAll(IntPtr fromList) { return false; } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + using var fp = Runtime.PySequence_GetItem(fromList, 0); + return Runtime.GetManagedString(fp) == "*"; } } } diff --git a/src/runtime/intern.cs b/src/runtime/intern.cs index d8bdf863e..ced1e5e92 100644 --- a/src/runtime/intern.cs +++ b/src/runtime/intern.cs @@ -42,9 +42,10 @@ public static void Initialize() public static void Shutdown() { - foreach (var ptr in _intern2strings.Keys) + foreach (var entry in _intern2strings) { - Runtime.XDecref(ptr); + Runtime.XDecref(entry.Key); + typeof(PyIdentifier).GetField(entry.Value).SetValue(null, IntPtr.Zero); } _string2interns = null; _intern2strings = null; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index ff9a98622..188db3a58 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -68,313 +68,48 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - public static int magic() => ManagedDataOffsets.Magic; - } - - internal static class ManagedDataOffsets - { - public static int Magic { get; internal set; } - public static readonly Dictionary NameMapping = new Dictionary(); - - static class DataOffsets - { - public static readonly int ob_data = 0; - public static readonly int ob_dict = 0; - - static DataOffsets() - { - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fields.Length; i++) - { - fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); - } - } - } - - static ManagedDataOffsets() - { - NameMapping = TypeOffset.GetOffsets(); - - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - size = fields.Length * IntPtr.Size; - } - - public static int GetSlotOffset(string name) - { - return NameMapping[name]; - } - - private static int BaseOffset(IntPtr type) - { - Debug.Assert(type != IntPtr.Zero); - int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0); - return typeSize; - } - - public static int DataOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_data; - } - - public static int DictOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_dict; - } - - public static int ob_data => DataOffsets.ob_data; - public static int ob_dict => DataOffsets.ob_dict; - public static int Size { get { return size; } } - - private static readonly int size; - } - - internal static class OriginalObjectOffsets - { - static OriginalObjectOffsets() - { - int size = IntPtr.Size; - var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD -#if PYTHON_WITH_PYDEBUG - _ob_next = 0; - _ob_prev = 1 * size; - n = 2; -#endif - ob_refcnt = (n + 0) * size; - ob_type = (n + 1) * size; - } - - public static int Size { get { return size; } } - - private static readonly int size = -#if PYTHON_WITH_PYDEBUG - 4 * IntPtr.Size; -#else - 2 * IntPtr.Size; -#endif - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset - { - static ObjectOffset() - { -#if PYTHON_WITH_PYDEBUG - _ob_next = OriginalObjectOffsets._ob_next; - _ob_prev = OriginalObjectOffsets._ob_prev; -#endif - ob_refcnt = OriginalObjectOffsets.ob_refcnt; - ob_type = OriginalObjectOffsets.ob_type; - - size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int magic(IntPtr type) - { - return ManagedDataOffsets.DataOffset(type); - } - - public static int TypeDictOffset(IntPtr type) - { - return ManagedDataOffsets.DictOffset(type); - } - - public static int Size(IntPtr pyType) - { - if (IsException(pyType)) - { - return ExceptionOffset.Size(); - } - - return size; - } - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - private static readonly int size; - - private static bool IsException(IntPtr pyObject) - { - var type = Runtime.PyObject_TYPE(pyObject); - return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) - || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - } - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ExceptionOffset - { - static ExceptionOffset() - { - Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); - } - - size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int Size() { return size; } - - // PyException_HEAD - // (start after PyObject_HEAD) - public static int dict = 0; - public static int args = 0; - public static int traceback = 0; - public static int context = 0; - public static int cause = 0; - public static int suppress_context = 0; - - private static readonly int size; - } - - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class BytesOffset - { - static BytesOffset() - { - Type type = typeof(BytesOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - /* The *real* layout of a type object when allocated on the heap */ - //typedef struct _heaptypeobject { -#if PYTHON_WITH_PYDEBUG -/* _PyObject_HEAD_EXTRA defines pointers to support a doubly-linked list of all live heap objects. */ - public static int _ob_next = 0; - public static int _ob_prev = 0; -#endif - // PyObject_VAR_HEAD { - // PyObject_HEAD { - public static int ob_refcnt = 0; - public static int ob_type = 0; - // } - public static int ob_size = 0; /* Number of items in _VAR_iable part */ - // } - public static int ob_shash = 0; - public static int ob_sval = 0; /* start of data */ - - /* Invariants: - * ob_sval contains space for 'ob_size+1' elements. - * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. - */ - //} PyBytesObject; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ModuleDefOffset - { - static ModuleDefOffset() - { - Type type = typeof(ModuleDefOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * size) + TypeOffset.ob_size); - } - } - - public static IntPtr AllocModuleDef(string modulename) - { - byte[] ascii = Encoding.ASCII.GetBytes(modulename); - int size = name + ascii.Length + 1; - IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i <= m_free; i += IntPtr.Size) - Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); - Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); - Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); - Marshal.WriteByte(ptr, name + ascii.Length, 0); - return ptr; - } - - public static void FreeModuleDef(IntPtr ptr) - { - Marshal.FreeHGlobal(ptr); - } - - // typedef struct PyModuleDef{ - // typedef struct PyModuleDef_Base { - // starts after PyObject_HEAD (TypeOffset.ob_type + 1) - public static int m_init = 0; - public static int m_index = 0; - public static int m_copy = 0; - // } PyModuleDef_Base - public static int m_name = 0; - public static int m_doc = 0; - public static int m_size = 0; - public static int m_methods = 0; - public static int m_reload = 0; - public static int m_traverse = 0; - public static int m_clear = 0; - public static int m_free = 0; - // } PyModuleDef - - public static int name = 0; - } - - /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. /// Note that the two values reserved for stackless have been put /// to good use as PythonNet specific flags (Managed and Subclass) /// - internal class TypeFlags + // Py_TPFLAGS_* + [Flags] + public enum TypeFlags: int { - public const int HeapType = (1 << 9); - public const int BaseType = (1 << 10); - public const int Ready = (1 << 12); - public const int Readying = (1 << 13); - public const int HaveGC = (1 << 14); + HeapType = (1 << 9), + BaseType = (1 << 10), + Ready = (1 << 12), + Readying = (1 << 13), + HaveGC = (1 << 14), // 15 and 16 are reserved for stackless - public const int HaveStacklessExtension = 0; + HaveStacklessExtension = 0, /* XXX Reusing reserved constants */ - public const int Managed = (1 << 15); // PythonNet specific - public const int Subclass = (1 << 16); // PythonNet specific - public const int HaveIndex = (1 << 17); + /// PythonNet specific + HasClrInstance = (1 << 15), + /// PythonNet specific + Subclass = (1 << 16), + HaveIndex = (1 << 17), /* Objects support nb_index in PyNumberMethods */ - public const int HaveVersionTag = (1 << 18); - public const int ValidVersionTag = (1 << 19); - public const int IsAbstract = (1 << 20); - public const int HaveNewBuffer = (1 << 21); + HaveVersionTag = (1 << 18), + ValidVersionTag = (1 << 19), + IsAbstract = (1 << 20), + HaveNewBuffer = (1 << 21), // TODO: Implement FastSubclass functions - public const int IntSubclass = (1 << 23); - public const int LongSubclass = (1 << 24); - public const int ListSubclass = (1 << 25); - public const int TupleSubclass = (1 << 26); - public const int StringSubclass = (1 << 27); - public const int UnicodeSubclass = (1 << 28); - public const int DictSubclass = (1 << 29); - public const int BaseExceptionSubclass = (1 << 30); - public const int TypeSubclass = (1 << 31); - - public const int Default = ( + IntSubclass = (1 << 23), + LongSubclass = (1 << 24), + ListSubclass = (1 << 25), + TupleSubclass = (1 << 26), + StringSubclass = (1 << 27), + UnicodeSubclass = (1 << 28), + DictSubclass = (1 << 29), + BaseExceptionSubclass = (1 << 30), + TypeSubclass = (1 << 31), + + Default = ( HaveStacklessExtension | - HaveVersionTag); + HaveVersionTag), } diff --git a/src/runtime/loader.cs b/src/runtime/loader.cs new file mode 100644 index 000000000..d5f31b247 --- /dev/null +++ b/src/runtime/loader.cs @@ -0,0 +1,83 @@ +using System.Diagnostics; +using System; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace Python.Runtime +{ + using static Runtime; + + [Obsolete("Only to be used from within Python")] + static class Loader + { + public unsafe static int Initialize(IntPtr data, int size) + { + IntPtr gs = IntPtr.Zero; + try + { + var dllPath = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + + if (!string.IsNullOrEmpty(dllPath)) + { + PythonDLL = dllPath; + } + else + { + PythonDLL = null; + } + + gs = PyGILState_Ensure(); + + // Console.WriteLine("Startup thread"); + PythonEngine.InitExt(); + // Console.WriteLine("Startup finished"); + } + catch (Exception exc) + { + Console.Error.Write( + $"Failed to initialize pythonnet: {exc}\n{exc.StackTrace}" + ); + return 1; + } + finally + { + if (gs != IntPtr.Zero) + { + PyGILState_Release(gs); + } + } + return 0; + } + + public unsafe static int Shutdown(IntPtr data, int size) + { + IntPtr gs = IntPtr.Zero; + try + { + var command = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + + if (command == "full_shutdown") + { + gs = PyGILState_Ensure(); + PythonEngine.Shutdown(); + } + } + catch (Exception exc) + { + Console.Error.Write( + $"Failed to shutdown pythonnet: {exc}\n{exc.StackTrace}" + ); + return 1; + } + finally + { + if (gs != IntPtr.Zero) + { + PyGILState_Release(gs); + } + } + return 0; + } + } +} diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index b4baef835..e2f042bb8 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics; @@ -27,6 +28,9 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + internal BorrowedReference ObjectReference => new(pyHandle); + internal BorrowedReference TypeReference => new(tpHandle); + private static readonly Dictionary _managedObjs = new Dictionary(); internal void IncrRefCount() @@ -75,12 +79,16 @@ internal void FreeGCHandle() } } - internal static ManagedType GetManagedObject(BorrowedReference ob) + /// + /// Given a Python object, return the associated managed object or null. + /// + internal static ManagedType? GetManagedObject(BorrowedReference ob) => GetManagedObject(ob.DangerousGetAddress()); + /// /// Given a Python object, return the associated managed object or null. /// - internal static ManagedType GetManagedObject(IntPtr ob) + internal static ManagedType? GetManagedObject(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -90,18 +98,11 @@ internal static ManagedType GetManagedObject(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); + if ((flags & TypeFlags.HasClrInstance) != 0) { - IntPtr op = tp == ob - ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); - if (op == IntPtr.Zero) - { - return null; - } - var gc = (GCHandle)op; - return (ManagedType)gc.Target; + var gc = TryGetGCHandle(new BorrowedReference(ob)); + return (ManagedType)gc?.Target; } } return null; @@ -110,35 +111,24 @@ internal static ManagedType GetManagedObject(IntPtr ob) /// /// Given a Python object, return the associated managed object type or null. /// - internal static ManagedType GetManagedObjectType(IntPtr ob) + internal static ManagedType? GetManagedObjectType(IntPtr ob) { if (ob != IntPtr.Zero) { IntPtr tp = Runtime.PyObject_TYPE(ob); - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); + if ((flags & TypeFlags.HasClrInstance) != 0) { - tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - var gc = (GCHandle)tp; + var gc = GetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType); return (ManagedType)gc.Target; } } return null; } - - internal static ManagedType GetManagedObjectErr(IntPtr ob) - { - ManagedType result = GetManagedObject(ob); - if (result == null) - { - Exceptions.SetError(Exceptions.TypeError, "invalid argument, expected CLR type"); - } - return result; - } - - - internal static bool IsManagedType(IntPtr ob) + internal static bool IsInstanceOfManagedType(BorrowedReference ob) + => IsInstanceOfManagedType(ob.DangerousGetAddressOrNull()); + internal static bool IsInstanceOfManagedType(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -148,18 +138,22 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - return true; - } + return IsManagedType(new BorrowedReference(tp)); } return false; } - public bool IsTypeObject() + internal static bool IsManagedType(BorrowedReference type) + { + var flags = (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags); + return (flags & TypeFlags.HasClrInstance) != 0; + } + + public bool IsClrMetaTypeInstance() { - return pyHandle == tpHandle; + Debug.Assert(Runtime.PyCLRMetaType != IntPtr.Zero); + Debug.Assert(pyHandle != IntPtr.Zero); + return Runtime.PyObject_TYPE(pyHandle) == Runtime.PyCLRMetaType; } internal static IDictionary GetManagedObjects() @@ -191,7 +185,8 @@ internal void CallTypeClear() { return; } - var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + + var clearPtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_clear); if (clearPtr == IntPtr.Zero) { return; @@ -209,7 +204,7 @@ internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) { return; } - var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + var traversePtr = Runtime.PyType_GetSlot(TypeReference, TypeSlotID.tp_traverse); if (traversePtr == IntPtr.Zero) { return; @@ -252,13 +247,81 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { IntPtr type = Runtime.PyObject_TYPE(ob); - return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + return Marshal.ReadIntPtr(ob, instanceDictOffset); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { IntPtr type = Runtime.PyObject_TYPE(ob); - Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + Marshal.WriteIntPtr(ob, instanceDictOffset, value); + } + + internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) + { + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || type == Runtime.CLRMetaType); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int gcHandleOffset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(gcHandleOffset > 0); + + handle = Marshal.ReadIntPtr(reflectedClrObject.DangerousGetAddress(), gcHandleOffset); + } + + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + GetGCHandle(reflectedClrObject, type, out IntPtr handle); + return handle == IntPtr.Zero ? null : (GCHandle)handle; + } + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject) + { + BorrowedReference reflectedType = Runtime.PyObject_TYPE(reflectedClrObject); + + return TryGetGCHandle(reflectedClrObject, reflectedType); + } + + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject) + => TryGetGCHandle(reflectedClrObject) ?? throw new InvalidOperationException(); + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + => TryGetGCHandle(reflectedClrObject, type) ?? throw new InvalidOperationException(); + + internal static void InitGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle handle) + { + Debug.Assert(TryGetGCHandle(reflectedClrObject) == null); + + SetGCHandle(reflectedClrObject, type: type, handle); + } + internal static void InitGCHandle(BorrowedReference reflectedClrObject, GCHandle handle) + => InitGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), handle); + + internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle newHandle) + { + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + Marshal.WriteIntPtr(reflectedClrObject.DangerousGetAddress(), offset, (IntPtr)newHandle); + } + internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) + => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); + + internal static class Offsets + { + static Offsets() + { + int pyTypeSize = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize); + if (pyTypeSize < 0) throw new InvalidOperationException(); + + tp_clr_inst_offset = pyTypeSize; + tp_clr_inst = tp_clr_inst_offset + IntPtr.Size; + } + public static int tp_clr_inst_offset { get; } + public static int tp_clr_inst { get; } } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 36b406c7b..014c5917c 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,6 +1,4 @@ using System; -using System.Collections; -using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -134,7 +132,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) { - return TypeManager.CreateSubType(name, base_type, dict); + return TypeManager.CreateSubType(name, base_type, clsDict.Reference); } } } @@ -147,13 +145,13 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; + var flags = TypeFlags.Default; + flags |= TypeFlags.HasClrInstance; flags |= TypeFlags.HeapType; flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); @@ -164,10 +162,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); + // derived types must have their GCHandle at the same offset as the base types + int clrInstOffset = Marshal.ReadInt32(base_type, Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, Offsets.tp_clr_inst_offset, clrInstOffset); // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); + IntPtr gc = Marshal.ReadIntPtr(base_type, Offsets.tp_clr_inst); + Marshal.WriteIntPtr(type, Offsets.tp_clr_inst, gc); + + if (Runtime.PyType_Ready(type) != 0) + throw PythonException.ThrowLastAsClrException(); return type; } @@ -205,6 +209,11 @@ public static IntPtr tp_call(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } + return CallInit(obj, args, kw); + } + + private static IntPtr CallInit(IntPtr obj, IntPtr args, IntPtr kw) + { var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); Runtime.PyErr_Clear(); @@ -257,7 +266,7 @@ public static int tp_setattro(IntPtr tp, IntPtr name, IntPtr value) } int res = Runtime.PyObject_GenericSetAttr(tp, name, value); - Runtime.PyType_Modified(tp); + Runtime.PyType_Modified(new BorrowedReference(tp)); return res; } @@ -285,11 +294,15 @@ public static void tp_dealloc(IntPtr tp) { // Fix this when we dont cheat on the handle for subclasses! - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { - IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - ((GCHandle)gc).Free(); + GetGCHandle(new BorrowedReference(tp)).Free(); +#if DEBUG + // prevent ExecutionEngineException in debug builds in case we have a bug + // this would allow using managed debugger to investigate the issue + SetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType, default); +#endif } IntPtr op = Marshal.ReadIntPtr(tp, TypeOffset.ob_type); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5de0ecc00..1b7cc4736 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -17,6 +17,9 @@ namespace Python.Runtime [Serializable] internal class MethodBinder { + /// + /// The overloads of this method + /// public List list; [NonSerialized] @@ -83,6 +86,7 @@ internal static MethodInfo MatchSignature(MethodInfo[] mi, Type[] tp) /// /// Given a sequence of MethodInfo and a sequence of type parameters, /// return the MethodInfo that represents the matching closed generic. + /// If unsuccessful, returns null and may set a Python error. /// internal static MethodInfo MatchParameters(MethodInfo[] mi, Type[] tp) { @@ -102,7 +106,18 @@ internal static MethodInfo MatchParameters(MethodInfo[] mi, Type[] tp) { continue; } - return t.MakeGenericMethod(tp); + try + { + // MakeGenericMethod can throw ArgumentException if the type parameters do not obey the constraints. + MethodInfo method = t.MakeGenericMethod(tp); + Exceptions.Clear(); + return method; + } + catch (ArgumentException e) + { + Exceptions.SetError(e); + // The error will remain set until cleared by a successful match. + } } return null; } @@ -277,20 +292,37 @@ internal static int ArgPrecedence(Type t) /// /// Bind the given Python instance and arguments to a particular method - /// overload and return a structure that contains the converted Python + /// overload in and return a structure that contains the converted Python /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw) { return Bind(inst, args, kw, null, null); } + /// + /// Bind the given Python instance and arguments to a particular method + /// overload in and return a structure that contains the converted Python + /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. + /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// If not null, only bind to that method. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { return Bind(inst, args, kw, info, null); } - private readonly struct MatchedMethod { + private readonly struct MatchedMethod + { public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int outs, MethodBase mb) { KwargsMatched = kwargsMatched; @@ -307,9 +339,33 @@ public MatchedMethod(int kwargsMatched, int defaultsNeeded, object[] margs, int public MethodBase Method { get; } } + private readonly struct MismatchedMethod + { + public MismatchedMethod(Exception exception, MethodBase mb) + { + Exception = exception; + Method = mb; + } + + public Exception Exception { get; } + public MethodBase Method { get; } + } + + /// + /// Bind the given Python instance and arguments to a particular method + /// overload in and return a structure that contains the converted Python + /// instance, converted arguments and the correct method to call. + /// If unsuccessful, may set a Python error. + /// + /// The Python target of the method invocation. + /// The Python arguments. + /// The Python keyword arguments. + /// If not null, only bind to that method. + /// If not null, additionally attempt to bind to the generic methods in this array by inferring generic type parameters. + /// A Binding if successful. Otherwise null. internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { - // loop to find match, return invoker w/ or /wo error + // loop to find match, return invoker w/ or w/o error MethodBase[] _methods = null; var kwargDict = new Dictionary(); @@ -340,6 +396,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } var argMatchedMethods = new List(_methods.Length); + var mismatchedMethods = new List(); // TODO: Clean up foreach (MethodBase mi in _methods) @@ -375,12 +432,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // We need to take the first CLR argument. pi = pi.Take(1).ToArray(); } - var outs = 0; + int outs; var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, needsResolution: _methods.Length > 1, // If there's more than one possible match. outs: out outs); if (margs == null) { + var mismatchCause = PythonException.FetchCurrent(); + mismatchedMethods.Add(new MismatchedMethod(mismatchCause, mi)); continue; } if (isOperator) @@ -403,7 +462,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } margs = margsTemp; } - else { break; } + else continue; } } @@ -434,6 +493,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // Best effort for determining method to match on gives multiple possible // matches and we need at least one default argument - bail from this point + StringBuilder stringBuilder = new StringBuilder("Not enough arguments provided to disambiguate the method. Found:"); + foreach (var matchedMethod in argMatchedMethods) + { + stringBuilder.AppendLine(); + stringBuilder.Append(matchedMethod.Method.ToString()); + } + Exceptions.SetError(Exceptions.TypeError, stringBuilder.ToString()); return null; } @@ -463,6 +529,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // XXX maybe better to do this before all the other rigmarole. if (co == null) { + Exceptions.SetError(Exceptions.TypeError, "Invoked a non-static method with an invalid instance"); return null; } target = co.inst; @@ -470,19 +537,32 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return new Binding(mi, target, margs, outs); } - // We weren't able to find a matching method but at least one - // is a generic method and info is null. That happens when a generic - // method was not called using the [] syntax. Let's introspect the - // type of the arguments and use it to construct the correct method. - if (isGeneric && info == null && methodinfo != null) + else if (isGeneric && info == null && methodinfo != null) { + // We weren't able to find a matching method but at least one + // is a generic method and info is null. That happens when a generic + // method was not called using the [] syntax. Let's introspect the + // type of the arguments and use it to construct the correct method. Type[] types = Runtime.PythonArgsToTypeArray(args, true); MethodInfo mi = MatchParameters(methodinfo, types); - return Bind(inst, args, kw, mi, null); + if (mi != null) + { + return Bind(inst, args, kw, mi, null); + } + } + if (mismatchedMethods.Count > 0) + { + var aggregateException = GetAggregateException(mismatchedMethods); + Exceptions.SetError(aggregateException); } return null; } + static AggregateException GetAggregateException(IEnumerable mismatchedMethods) + { + return new AggregateException(mismatchedMethods.Select(m => new ArgumentException($"{m.Exception.Message} in method {m.Method}", m.Exception))); + } + static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out bool isNewReference) { isNewReference = false; @@ -517,6 +597,7 @@ static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out /// /// Attempts to convert Python positional argument tuple and keyword argument table /// into an array of managed objects, that can be passed to a method. + /// If unsuccessful, returns null and may set a Python error. /// /// Information about expected parameters /// true, if the last parameter is a params array. @@ -526,7 +607,7 @@ static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out /// A list of default values for omitted parameters /// true, if overloading resolution is required /// Returns number of output parameters - /// An array of .NET arguments, that can be passed to a method. + /// If successful, an array of .NET arguments that can be passed to the method. Otherwise null. static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, IntPtr args, int pyArgCount, Dictionary kwargDict, @@ -541,7 +622,7 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) { var parameter = pi[paramIndex]; - bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + bool hasNamedParam = parameter.Name != null ? kwargDict.ContainsKey(parameter.Name) : false; bool isNewReference = false; if (paramIndex >= pyArgCount && !(hasNamedParam || (paramsArray && paramIndex == arrayStart))) @@ -596,13 +677,14 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, /// /// Try to convert a Python argument object to a managed CLR type. + /// If unsuccessful, may set a Python error. /// - /// Pointer to the object at a particular parameter. + /// Pointer to the Python argument object. /// That parameter's managed type. - /// There are multiple overloading methods that need resolution. + /// If true, there are multiple overloading methods that need resolution. /// Converted argument. /// Whether the CLR type is passed by reference. - /// + /// true on success static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, out object arg, out bool isOut) { @@ -614,9 +696,8 @@ static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResoluti return false; } - if (!Converter.ToManaged(op, clrtype, out arg, false)) + if (!Converter.ToManaged(op, clrtype, out arg, true)) { - Exceptions.Clear(); return false; } @@ -624,6 +705,13 @@ static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResoluti return true; } + /// + /// Determine the managed type that a Python argument object needs to be converted into. + /// + /// The parameter's managed type. + /// Pointer to the Python argument object. + /// If true, there are multiple overloading methods that need resolution. + /// null if conversion is not possible static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool needsResolution) { // this logic below handles cases when multiple overloading methods @@ -635,7 +723,6 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool { // HACK: each overload should be weighted in some way instead pyoptype = Runtime.PyObject_Type(argument); - Exceptions.Clear(); if (pyoptype != IntPtr.Zero) { clrtype = Converter.GetTypeByAlias(pyoptype); @@ -645,12 +732,11 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool if (clrtype != null) { - var typematch = false; if ((parameterType != typeof(object)) && (parameterType != clrtype)) { IntPtr pytype = Converter.GetPythonTypeByAlias(parameterType); pyoptype = Runtime.PyObject_Type(argument); - Exceptions.Clear(); + var typematch = false; if (pyoptype != IntPtr.Zero) { if (pytype != pyoptype) @@ -666,13 +752,17 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool if (!typematch) { // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(parameterType); - TypeCode paramtypecode = Type.GetTypeCode(clrtype); - if (argtypecode == paramtypecode) + TypeCode parameterTypeCode = Type.GetTypeCode(parameterType); + TypeCode clrTypeCode = Type.GetTypeCode(clrtype); + if (parameterTypeCode == clrTypeCode) { typematch = true; clrtype = parameterType; } + else + { + Exceptions.RaiseTypeError($"Expected {parameterTypeCode}, got {clrTypeCode}"); + } } Runtime.XDecref(pyoptype); if (!typematch) @@ -682,7 +772,6 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool } else { - typematch = true; clrtype = parameterType; } } @@ -787,7 +876,7 @@ protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) { try { - var description = Runtime.PyObject_Unicode(type); + var description = Runtime.PyObject_Str(type); if (description != IntPtr.Zero) { to.Append(Runtime.GetManagedString(description)); @@ -815,9 +904,9 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i var msg = new StringBuilder("The underlying C# method(s) have been deleted"); if (list.Count > 0 && list[0].Name != null) { - msg.Append($": {list[0].ToString()}"); + msg.Append($": {list[0]}"); } - return Exceptions.RaiseTypeError(msg.ToString());; + return Exceptions.RaiseTypeError(msg.ToString()); } Binding binding = Bind(inst, args, kw, info, methodinfo); @@ -831,10 +920,16 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i { value.Append($" for {methodinfo[0].Name}"); } + else if (list.Count > 0 && list[0].Valid) + { + value.Append($" for {list[0].Value.Name}"); + } value.Append(": "); + Runtime.PyErr_Fetch(out var errType, out var errVal, out var errTrace); AppendArgumentTypes(to: value, args); - Exceptions.SetError(Exceptions.TypeError, value.ToString()); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), errTrace.StealNullable()); + Exceptions.RaiseTypeError(value.ToString()); return IntPtr.Zero; } @@ -867,34 +962,35 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i } // If there are out parameters, we return a tuple containing - // the result followed by the out parameters. If there is only + // the result, if any, followed by the out parameters. If there is only // one out parameter and the return type of the method is void, // we return the out parameter as the result to Python (for // code compatibility with ironpython). var mi = (MethodInfo)binding.info; - if (binding.outs == 1 && mi.ReturnType == typeof(void)) - { - } - if (binding.outs > 0) { ParameterInfo[] pi = mi.GetParameters(); int c = pi.Length; var n = 0; - IntPtr t = Runtime.PyTuple_New(binding.outs + 1); - IntPtr v = Converter.ToPython(result, mi.ReturnType); - Runtime.PyTuple_SetItem(t, n, v); - n++; + bool isVoid = mi.ReturnType == typeof(void); + int tupleSize = binding.outs + (isVoid ? 0 : 1); + IntPtr t = Runtime.PyTuple_New(tupleSize); + if (!isVoid) + { + IntPtr v = Converter.ToPython(result, mi.ReturnType); + Runtime.PyTuple_SetItem(t, n, v); + n++; + } for (var i = 0; i < c; i++) { Type pt = pi[i].ParameterType; - if (pi[i].IsOut || pt.IsByRef) + if (pt.IsByRef) { - v = Converter.ToPython(binding.args[i], pt.GetElementType()); + IntPtr v = Converter.ToPython(binding.args[i], pt.GetElementType()); Runtime.PyTuple_SetItem(t, n, v); n++; } @@ -902,7 +998,7 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding.outs == 1 && mi.ReturnType == typeof(void)) { - v = Runtime.PyTuple_GetItem(t, 1); + IntPtr v = Runtime.PyTuple_GetItem(t, 0); Runtime.XIncref(v); Runtime.XDecref(t); return v; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 46b62807d..c1e729f9e 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -31,7 +31,7 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) { Runtime.XIncref(targetType); } - + this.targetType = targetType; this.info = null; @@ -42,12 +42,6 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref target); - Runtime.Py_CLEAR(ref targetType); - } - /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -201,35 +195,27 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) /// /// MethodBinding __hash__ implementation. /// - public static IntPtr tp_hash(IntPtr ob) + public static nint tp_hash(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - long x = 0; - long y = 0; + nint x = 0; if (self.target != IntPtr.Zero) { - x = Runtime.PyObject_Hash(self.target).ToInt64(); + x = Runtime.PyObject_Hash(self.target); if (x == -1) { - return new IntPtr(-1); + return x; } } - y = Runtime.PyObject_Hash(self.m.pyHandle).ToInt64(); + nint y = Runtime.PyObject_Hash(self.m.pyHandle); if (y == -1) { - return new IntPtr(-1); - } - - x ^= y; - - if (x == -1) - { - x = -1; + return y; } - return new IntPtr(x); + return x ^ y; } /// @@ -243,21 +229,11 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($"<{type} method '{name}'>"); } - /// - /// MethodBinding dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Clear() { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - self.Dealloc(); - } - - public static int tp_clear(IntPtr ob) - { - var self = (MethodBinding)GetManagedObject(ob); - self.ClearMembers(); - return 0; + Runtime.Py_CLEAR(ref this.target); + Runtime.Py_CLEAR(ref this.targetType); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 37c01f5c5..2787ec999 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -120,16 +120,6 @@ internal bool IsStatic() return is_static; } - private void ClearMembers() - { - Runtime.Py_CLEAR(ref doc); - if (unbound != null) - { - Runtime.XDecref(unbound.pyHandle); - unbound = null; - } - } - /// /// Descriptor __getattribute__ implementation. /// @@ -210,23 +200,17 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - /// - /// Descriptor dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Clear() { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - self.Dealloc(); - } + Runtime.Py_CLEAR(ref this.doc); + if (this.unbound != null) + { + Runtime.XDecref(this.unbound.pyHandle); + this.unbound = null; + } - public static int tp_clear(IntPtr ob) - { - var self = (MethodObject)GetManagedObject(ob); - self.ClearMembers(); - ClearObjectDict(ob); - return 0; + ClearObjectDict(this.pyHandle); + base.Clear(); } protected override void OnSave(InterDomainContext context) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 07dd20e55..c2614b1d8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -18,7 +18,14 @@ internal class ModuleObject : ExtensionType internal string moduleName; internal IntPtr dict; + internal BorrowedReference DictRef => new BorrowedReference(dict); protected string _namespace; + private IntPtr __all__ = IntPtr.Zero; + + // Attributes to be set on the module according to PEP302 and 451 + // by the import machinery. + static readonly HashSet settableAttributes = + new HashSet {"__spec__", "__file__", "__name__", "__path__", "__loader__", "__package__"}; public ModuleObject(string name) { @@ -43,21 +50,18 @@ public ModuleObject(string name) docstring += "- " + a.FullName + "\n"; } - dict = Runtime.PyDict_New(); - IntPtr pyname = Runtime.PyString_FromString(moduleName); - IntPtr pyfilename = Runtime.PyString_FromString(filename); - IntPtr pydocstring = Runtime.PyString_FromString(docstring); - IntPtr pycls = TypeManager.GetTypeHandle(GetType()); - Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname); - Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename); - Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring); - Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls); - Runtime.XDecref(pyname); - Runtime.XDecref(pyfilename); - Runtime.XDecref(pydocstring); - - Runtime.XIncref(dict); - SetObjectDict(pyHandle, dict); + var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); + PythonException.ThrowIfIsNull(dictRef); + dict = dictRef.DangerousMoveToPointer(); + __all__ = Runtime.PyList_New(0); + using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); + using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); + using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); + BorrowedReference pycls = TypeManager.GetTypeReference(GetType()); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__name__, pyname); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__file__, pyfilename); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__doc__, pydocstring); + Runtime.PyDict_SetItem(DictRef, PyIdentifier.__class__, pycls); InitializeModuleMembers(); } @@ -155,7 +159,7 @@ private void StoreAttribute(string name, ManagedType ob) { if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } ob.IncrRefCount(); cache[name] = ob; @@ -177,13 +181,29 @@ public void LoadNames() { continue; } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); + BorrowedReference attr = Runtime.PyDict_GetItemString(DictRef, name); // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) + if (!attr.IsNull) { continue; } - GetAttribute(name, true); + + if(GetAttribute(name, true) != null) + { + // if it's a valid attribute, add it to __all__ + var pyname = Runtime.PyString_FromString(name); + try + { + if (Runtime.PyList_Append(new BorrowedReference(__all__), new BorrowedReference(pyname)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + finally + { + Runtime.XDecref(pyname); + } + } } } @@ -265,6 +285,13 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } + if (name == "__all__") + { + self.LoadNames(); + Runtime.XIncref(self.__all__); + return self.__all__; + } + ManagedType attr = null; try @@ -276,7 +303,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) Exceptions.SetError(e); return IntPtr.Zero; } - + if (attr == null) { @@ -297,13 +324,6 @@ public static IntPtr tp_repr(IntPtr ob) return Runtime.PyString_FromString($""); } - public new static void tp_dealloc(IntPtr ob) - { - var self = (ModuleObject)GetManagedObject(ob); - tp_clear(ob); - self.Dealloc(); - } - public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ModuleObject)GetManagedObject(ob); @@ -317,17 +337,35 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - public static int tp_clear(IntPtr ob) + protected override void Clear() { - var self = (ModuleObject)GetManagedObject(ob); - Runtime.Py_CLEAR(ref self.dict); - ClearObjectDict(ob); - foreach (var attr in self.cache.Values) + Runtime.Py_CLEAR(ref this.dict); + ClearObjectDict(this.pyHandle); + foreach (var attr in this.cache.Values) { Runtime.XDecref(attr.pyHandle); } - self.cache.Clear(); - return 0; + this.cache.Clear(); + base.Clear(); + } + + /// + /// Override the setattr implementation. + /// This is needed because the import mechanics need + /// to set a few attributes + /// + [ForbidPythonThreads] + public new static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) + { + var managedKey = Runtime.GetManagedString(key); + if ((settableAttributes.Contains(managedKey)) || + (ManagedType.GetManagedObject(val)?.GetType() == typeof(ModuleObject)) ) + { + var self = (ModuleObject)ManagedType.GetManagedObject(ob); + return Runtime.PyDict_SetItem(self.dict, key, val); + } + + return ExtensionType.tp_setattro(ob, key, val); } protected override void OnSave(InterDomainContext context) @@ -344,16 +382,16 @@ protected override void OnSave(InterDomainContext context) // destroy the cache(s) foreach (var pair in cache) { - if ((Runtime.PyDict_DelItemString(dict, pair.Key) == -1) && + if ((Runtime.PyDict_DelItemString(DictRef, pair.Key) == -1) && (Exceptions.ExceptionMatches(Exceptions.KeyError))) { - // Trying to remove a key that's not in the dictionary + // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. Runtime.PyErr_Clear(); } else if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } pair.Value.DecrRefCount(); } @@ -471,6 +509,7 @@ public static bool SuppressOverloads public static Assembly AddReference(string name) { AssemblyManager.UpdatePath(); + var origNs = AssemblyManager.GetNamespaces(); Assembly assembly = null; assembly = AssemblyManager.FindLoadedAssembly(name); if (assembly == null) @@ -489,7 +528,16 @@ public static Assembly AddReference(string name) { throw new FileNotFoundException($"Unable to find assembly '{name}'."); } - + // Classes that are not in a namespace needs an extra nudge to be found. + ImportHook.UpdateCLRModuleDict(); + + // A bit heavyhanded, but we can't use the AssemblyManager's AssemblyLoadHandler + // method because it may be called from other threads, leading to deadlocks + // if it is called while Python code is executing. + var currNs = AssemblyManager.GetNamespaces().Except(origNs); + foreach(var ns in currNs){ + ImportHook.AddNamespace(ns); + } return assembly; } @@ -498,7 +546,7 @@ public static Assembly AddReference(string name) /// clr.GetClrType(IComparable) gives you the Type for IComparable, /// that you can e.g. perform reflection on. Similar to typeof(IComparable) in C# /// or clr.GetClrType(IComparable) in IronPython. - /// + /// /// /// /// The Type object @@ -537,10 +585,22 @@ public static string[] ListAssemblies(bool verbose) return names; } + /// + /// Note: This should *not* be called directly. + /// The function that get/import a CLR assembly as a python module. + /// This function should only be called by the import machinery as seen + /// in importhook.cs + /// + /// A ModuleSpec Python object + /// A new reference to the imported module, as a PyObject. [ModuleFunction] - public static int _AtExit() + [ForbidPythonThreads] + public static ModuleObject _load_clr_module(PyObject spec) { - return Runtime.AtExit(); + ModuleObject mod = null; + using var modname = spec.GetAttr("name"); + mod = ImportHook.Import(modname.ToString()); + return mod; } } } diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index e95b259c5..e99fc33ab 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -4,10 +4,12 @@ namespace Python.Runtime.Native using System.Globalization; using System.Linq; using System.Reflection; - using System.Runtime.InteropServices; static class ABI { + public static int RefCountOffset { get; } = GetRefCountOffset(); + public static int ObjectHeadOffset => RefCountOffset; + internal static void Initialize(Version version, BorrowedReference pyType) { string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture, @@ -17,21 +19,34 @@ internal static void Initialize(Version version, BorrowedReference pyType) const string nativeTypeOffsetClassName = "Python.Runtime.NativeTypeOffset"; string className = "Python.Runtime.TypeOffset" + offsetsClassSuffix; + Type nativeOffsetsClass = thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false); Type typeOffsetsClass = // Try platform native offsets first. It is only present when generated by setup.py - thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false) - ?? thisAssembly.GetType(className, throwOnError: false); + nativeOffsetsClass ?? thisAssembly.GetType(className, throwOnError: false); if (typeOffsetsClass is null) { var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset")); - string message = $"Searching for {className}, found {string.Join(",", types)}. " + - "If you are building Python.NET from source, make sure you have run 'python setup.py configure' to fill in configured.props"; + string message = $"Searching for {className}, found {string.Join(",", types)}."; throw new NotSupportedException($"Python ABI v{version} is not supported: {message}"); } + var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); - TypeOffset.Use(typeOffsets); + TypeOffset.Use(typeOffsets, nativeOffsetsClass == null ? ObjectHeadOffset : 0); + } - ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); + static unsafe int GetRefCountOffset() + { + IntPtr tempObject = Runtime.PyList_New(0); + IntPtr* tempPtr = (IntPtr*)tempObject; + int offset = 0; + while(tempPtr[offset] != (IntPtr)1) + { + offset++; + if (offset > 100) + throw new InvalidProgramException("PyObject_HEAD could not be found withing reasonable distance from the start of PyObject"); + } + Runtime.XDecref(tempObject); + return offset * IntPtr.Size; } } } diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 485c041f8..0829e5bc9 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -63,6 +63,7 @@ interface ITypeOffsets int tp_new { get; } int tp_repr { get; } int tp_richcompare { get; } + int tp_weaklistoffset { get; } int tp_setattro { get; } int tp_str { get; } int tp_traverse { get; } diff --git a/src/runtime/native/NativeTypeSpec.cs b/src/runtime/native/NativeTypeSpec.cs new file mode 100644 index 000000000..d55b77381 --- /dev/null +++ b/src/runtime/native/NativeTypeSpec.cs @@ -0,0 +1,45 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime.Native +{ + [StructLayout(LayoutKind.Sequential)] + struct NativeTypeSpec : IDisposable + { + public readonly StrPtr Name; + public readonly int BasicSize; + public readonly int ItemSize; + public readonly TypeFlags Flags; + public IntPtr Slots; + + public NativeTypeSpec(TypeSpec spec) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.BasicSize = spec.BasicSize; + this.ItemSize = spec.ItemSize; + this.Flags = spec.Flags; + + unsafe + { + int slotsBytes = checked((spec.Slots.Count + 1) * Marshal.SizeOf()); + var slots = (TypeSpec.Slot*)Marshal.AllocHGlobal(slotsBytes); + for (int slotIndex = 0; slotIndex < spec.Slots.Count; slotIndex++) + slots[slotIndex] = spec.Slots[slotIndex]; + slots[spec.Slots.Count] = default; + this.Slots = (IntPtr)slots; + } + } + + public void Dispose() + { + // we have to leak the name + // this.Name.Dispose(); + Marshal.FreeHGlobal(this.Slots); + this.Slots = IntPtr.Zero; + } + } +} diff --git a/src/runtime/native/PyCompilerFlags.cs b/src/runtime/native/PyCompilerFlags.cs new file mode 100644 index 000000000..9e8242c58 --- /dev/null +++ b/src/runtime/native/PyCompilerFlags.cs @@ -0,0 +1,13 @@ +using System; + +namespace Python.Runtime.Native +{ + [Flags] + enum PyCompilerFlags + { + SOURCE_IS_UTF8 = 0x0100, + DONT_IMPLY_DEDENT = 0x0200, + ONLY_AST = 0x0400, + IGNORE_COOKIE = 0x0800, + } +} diff --git a/src/runtime/native/StrPtr.cs b/src/runtime/native/StrPtr.cs new file mode 100644 index 000000000..99aa35ddd --- /dev/null +++ b/src/runtime/native/StrPtr.cs @@ -0,0 +1,73 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime.Native +{ + [StructLayout(LayoutKind.Sequential)] + struct StrPtr : IDisposable + { + public IntPtr RawPointer { get; set; } + unsafe byte* Bytes => (byte*)this.RawPointer; + + public unsafe StrPtr(string value, Encoding encoding) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + if (encoding is null) throw new ArgumentNullException(nameof(encoding)); + + var bytes = encoding.GetBytes(value); + this.RawPointer = Marshal.AllocHGlobal(checked(bytes.Length + 1)); + try + { + Marshal.Copy(bytes, 0, this.RawPointer, bytes.Length); + this.Bytes[bytes.Length] = 0; + } + catch + { + this.Dispose(); + throw; + } + } + + public unsafe string? ToString(Encoding encoding) + { + if (encoding is null) throw new ArgumentNullException(nameof(encoding)); + if (this.RawPointer == IntPtr.Zero) return null; + + return encoding.GetString((byte*)this.RawPointer, byteCount: checked((int)this.ByteCount)); + } + + public unsafe nuint ByteCount + { + get + { + if (this.RawPointer == IntPtr.Zero) throw new NullReferenceException(); + + nuint zeroIndex = 0; + while (this.Bytes[zeroIndex] != 0) + { + zeroIndex++; + } + return zeroIndex; + } + } + + public void Dispose() + { + if (this.RawPointer == IntPtr.Zero) + return; + + Marshal.FreeHGlobal(this.RawPointer); + this.RawPointer = IntPtr.Zero; + } + + internal static Encoding GetEncodingByPythonName(string pyEncodingName) + { + // https://stackoverflow.com/a/7798749/231238 + if (pyEncodingName == "mbcs") return Encoding.Default; + + return Encoding.GetEncoding(pyEncodingName); + } + } +} diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 4c1bcefa0..6e6da2d93 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -70,11 +70,12 @@ static partial class TypeOffset internal static int tp_new { get; private set; } internal static int tp_repr { get; private set; } internal static int tp_richcompare { get; private set; } + internal static int tp_weaklistoffset { get; private set; } internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } - internal static void Use(ITypeOffsets offsets) + internal static void Use(ITypeOffsets offsets, int extraHeadOffset) { if (offsets is null) throw new ArgumentNullException(nameof(offsets)); @@ -86,14 +87,19 @@ internal static void Use(ITypeOffsets offsets) var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name); int value = (int)sourceProperty.GetValue(offsets, null); + value += extraHeadOffset; offsetProperty.SetValue(obj: null, value: value, index: null); } ValidateUnusedTypeOffsetProperties(offsetProperties); ValidateRequiredOffsetsPresent(offsetProperties); + + SlotOffsets = GetOffsets(); } static readonly BindingFlags FieldFlags = BindingFlags.NonPublic | BindingFlags.Static; + + static Dictionary SlotOffsets; internal static Dictionary GetOffsets() { var properties = typeof(TypeOffset).GetProperties(FieldFlags); @@ -104,10 +110,9 @@ internal static Dictionary GetOffsets() return result; } - internal static int GetOffsetUncached(string name) + public static int GetSlotOffset(string slotName) { - var property = typeof(TypeOffset).GetProperty(name, FieldFlags); - return (int)property.GetValue(obj: null, index: null); + return SlotOffsets[slotName]; } static readonly HashSet slotNames = new HashSet(); @@ -147,9 +152,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) { "__instancecheck__", "__subclasscheck__", - "_AtExit", "AddReference", - "FinalizeObject", "FindAssembly", "get_SuppressDocs", "get_SuppressOverloads", @@ -157,6 +160,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "getPreload", "Initialize", "ListAssemblies", + "_load_clr_module", "Release", "Reset", "set_SuppressDocs", diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs index 59bf944bc..e44dc3be1 100644 --- a/src/runtime/operatormethod.cs +++ b/src/runtime/operatormethod.cs @@ -51,7 +51,6 @@ static OperatorMethod() ["op_OnesComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), ["op_UnaryNegation"] = new SlotDefinition("__neg__", TypeOffset.nb_negative), ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), - ["op_OneComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), }; ComparisonOpMap = new Dictionary { @@ -80,7 +79,7 @@ public static void Shutdown() public static bool IsOperatorMethod(MethodBase method) { - if (!method.IsSpecialName) + if (!method.IsSpecialName && !method.IsOpsHelper()) { return false; } @@ -102,7 +101,12 @@ public static void FixupSlots(IntPtr pyType, Type clrType) { const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; Debug.Assert(_opType != null); - foreach (var method in clrType.GetMethods(flags)) + + var staticMethods = + clrType.IsEnum ? typeof(EnumOps<>).MakeGenericType(clrType).GetMethods(flags) + : clrType.GetMethods(flags); + + foreach (var method in staticMethods) { // We only want to override slots for operators excluding // comparison operators, which are handled by ClassBase.tp_richcompare. @@ -170,9 +174,11 @@ public static string ReversePyMethodName(string pyName) /// public static bool IsReverse(MethodInfo method) { - Type declaringType = method.DeclaringType; + Type primaryType = method.IsOpsHelper() + ? method.DeclaringType.GetGenericArguments()[0] + : method.DeclaringType; Type leftOperandType = method.GetParameters()[0].ParameterType; - return leftOperandType != declaringType; + return leftOperandType != primaryType; } public static void FilterMethods(MethodInfo[] methods, out MethodInfo[] forwardMethods, out MethodInfo[] reverseMethods) diff --git a/src/runtime/opshelper.cs b/src/runtime/opshelper.cs new file mode 100644 index 000000000..59f7704b7 --- /dev/null +++ b/src/runtime/opshelper.cs @@ -0,0 +1,77 @@ +using System; +using System.Linq.Expressions; +using System.Reflection; + +using static Python.Runtime.OpsHelper; + +namespace Python.Runtime +{ + static class OpsHelper + { + public static BindingFlags BindingFlags => BindingFlags.Public | BindingFlags.Static; + + public static Func Binary(Func func) + { + var a = Expression.Parameter(typeof(T), "a"); + var b = Expression.Parameter(typeof(T), "b"); + var body = func(a, b); + var lambda = Expression.Lambda>(body, a, b); + return lambda.Compile(); + } + + public static Func Unary(Func func) + { + var value = Expression.Parameter(typeof(T), "value"); + var body = func(value); + var lambda = Expression.Lambda>(body, value); + return lambda.Compile(); + } + + public static bool IsOpsHelper(this MethodBase method) + => method.DeclaringType.GetCustomAttribute() is not null; + + public static Expression EnumUnderlyingValue(Expression enumValue) + => Expression.Convert(enumValue, enumValue.Type.GetEnumUnderlyingType()); + } + + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + internal class OpsAttribute: Attribute { } + + [Ops] + internal static class EnumOps where T : Enum + { + static readonly Func and = BinaryOp(Expression.And); + static readonly Func or = BinaryOp(Expression.Or); + static readonly Func xor = BinaryOp(Expression.ExclusiveOr); + + static readonly Func invert = UnaryOp(Expression.OnesComplement); + + public static T op_BitwiseAnd(T a, T b) => and(a, b); + public static T op_BitwiseOr(T a, T b) => or(a, b); + public static T op_ExclusiveOr(T a, T b) => xor(a, b); + public static T op_OnesComplement(T value) => invert(value); + + static Expression FromNumber(Expression number) + => Expression.Convert(number, typeof(T)); + + static Func BinaryOp(Func op) + { + return Binary((a, b) => + { + var numericA = EnumUnderlyingValue(a); + var numericB = EnumUnderlyingValue(b); + var numericResult = op(numericA, numericB); + return FromNumber(numericResult); + }); + } + static Func UnaryOp(Func op) + { + return Unary(value => + { + var numeric = EnumUnderlyingValue(value); + var numericResult = op(numeric); + return FromNumber(numericResult); + }); + } + } +} diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index e9fa91d3b..8222dc136 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -58,14 +58,10 @@ public static IntPtr tp_repr(IntPtr op) return doc; } - /// - /// OverloadMapper dealloc implementation. - /// - public new static void tp_dealloc(IntPtr ob) + protected override void Clear() { - var self = (OverloadMapper)GetManagedObject(ob); - Runtime.XDecref(self.target); - self.Dealloc(); + Runtime.Py_CLEAR(ref this.target); + base.Clear(); } } } diff --git a/src/runtime/platform/LibDL.cs b/src/runtime/platform/LibDL.cs new file mode 100644 index 000000000..3bf2a2057 --- /dev/null +++ b/src/runtime/platform/LibDL.cs @@ -0,0 +1,109 @@ +#pragma warning disable IDE1006 // Naming Styles (interface for native functions) +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibDL + { + IntPtr dlopen(string fileName, int flags); + IntPtr dlsym(IntPtr handle, string symbol); + int dlclose(IntPtr handle); + IntPtr dlerror(); + + int RTLD_NOW { get; } + int RTLD_GLOBAL { get; } + IntPtr RTLD_DEFAULT { get; } + } + + class LinuxLibDL : ILibDL + { + private const string NativeDll = "libdl.so"; + + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x100; + public IntPtr RTLD_DEFAULT => IntPtr.Zero; + + public static ILibDL GetInstance() + { + try + { + ILibDL libdl2 = new LinuxLibDL2(); + // call dlerror to ensure library is resolved + libdl2.dlerror(); + return libdl2; + } catch (DllNotFoundException) + { + return new LinuxLibDL(); + } + } + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class LinuxLibDL2 : ILibDL + { + private const string NativeDll = "libdl.so.2"; + + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x100; + public IntPtr RTLD_DEFAULT => IntPtr.Zero; + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class MacLibDL : ILibDL + { + public int RTLD_NOW => 0x2; + public int RTLD_GLOBAL => 0x8; + const string NativeDll = "/usr/lib/libSystem.dylib"; + public IntPtr RTLD_DEFAULT => new(-2); + + IntPtr ILibDL.dlopen(string fileName, int flags) => dlopen(fileName, flags); + IntPtr ILibDL.dlsym(IntPtr handle, string symbol) => dlsym(handle, symbol); + int ILibDL.dlclose(IntPtr handle) => dlclose(handle); + IntPtr ILibDL.dlerror() => dlerror(); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } +} diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index 193dff274..e361f87e4 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -1,5 +1,7 @@ using System; using System.ComponentModel; +using System.Diagnostics; +using System.Linq; using System.Runtime.InteropServices; namespace Python.Runtime.Platform @@ -26,9 +28,9 @@ public static ILibraryLoader Instance if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) _instance = new WindowsLoader(); else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - _instance = new LinuxLoader(); + _instance = new PosixLoader(LinuxLibDL.GetInstance()); else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - _instance = new DarwinLoader(); + _instance = new PosixLoader(new MacLibDL()); else throw new PlatformNotSupportedException( "This operating system is not supported" @@ -40,93 +42,23 @@ public static ILibraryLoader Instance } } - class LinuxLoader : ILibraryLoader + class PosixLoader : ILibraryLoader { - private static int RTLD_NOW = 0x2; - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; + private readonly ILibDL libDL; - public IntPtr Load(string dllToLoad) - { - var filename = $"lib{dllToLoad}.so"; - ClearError(); - var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); - if (res == IntPtr.Zero) - { - var err = GetError(); - throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); - } - - return res; - } - - public void Free(IntPtr handle) - { - dlclose(handle); - } - - public IntPtr GetFunction(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - ClearError(); - IntPtr res = dlsym(dllHandle, name); - if (res == IntPtr.Zero) - { - var err = GetError(); - throw new MissingMethodException($"Failed to load symbol {name}: {err}"); - } - return res; - } - - void ClearError() + public PosixLoader(ILibDL libDL) { - dlerror(); + this.libDL = libDL ?? throw new ArgumentNullException(nameof(libDL)); } - string GetError() - { - var res = dlerror(); - if (res != IntPtr.Zero) - return Marshal.PtrToStringAnsi(res); - else - return null; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(string fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, string symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); - } - - class DarwinLoader : ILibraryLoader - { - private static int RTLD_NOW = 0x2; - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - public IntPtr Load(string dllToLoad) { - var filename = $"lib{dllToLoad}.dylib"; ClearError(); - var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + var res = libDL.dlopen(dllToLoad, libDL.RTLD_NOW | libDL.RTLD_GLOBAL); if (res == IntPtr.Zero) { var err = GetError(); - throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + throw new DllNotFoundException($"Could not load {dllToLoad} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); } return res; @@ -134,7 +66,7 @@ public IntPtr Load(string dllToLoad) public void Free(IntPtr handle) { - dlclose(handle); + libDL.dlclose(handle); } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -142,11 +74,11 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) // look in the exe if dllHandle is NULL if (dllHandle == IntPtr.Zero) { - dllHandle = RTLD_DEFAULT; + dllHandle = libDL.RTLD_DEFAULT; } ClearError(); - IntPtr res = dlsym(dllHandle, name); + IntPtr res = libDL.dlsym(dllHandle, name); if (res == IntPtr.Zero) { var err = GetError(); @@ -157,29 +89,17 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) void ClearError() { - dlerror(); + libDL.dlerror(); } string GetError() { - var res = dlerror(); + var res = libDL.dlerror(); if (res != IntPtr.Zero) return Marshal.PtrToStringAnsi(res); else return null; } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); } class WindowsLoader : ILibraryLoader @@ -197,6 +117,15 @@ public IntPtr Load(string dllToLoad) public IntPtr GetFunction(IntPtr hModule, string procedureName) { + if (hModule == IntPtr.Zero) + { + foreach(var module in GetAllModules()) + { + var func = GetProcAddress(module, procedureName); + if (func != IntPtr.Zero) return func; + } + } + var res = WindowsLoader.GetProcAddress(hModule, procedureName); if (res == IntPtr.Zero) throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); @@ -205,6 +134,24 @@ public IntPtr GetFunction(IntPtr hModule, string procedureName) public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + static IntPtr[] GetAllModules() + { + var self = Process.GetCurrentProcess().Handle; + + uint bytes = 0; + var result = new IntPtr[0]; + if (!EnumProcessModules(self, result, bytes, out var needsBytes)) + throw new Win32Exception(); + while (bytes < needsBytes) + { + bytes = needsBytes; + result = new IntPtr[bytes / IntPtr.Size]; + if (!EnumProcessModules(self, result, bytes, out needsBytes)) + throw new Win32Exception(); + } + return result.Take((int)(needsBytes / IntPtr.Size)).ToArray(); + } + [DllImport(NativeDll, SetLastError = true)] static extern IntPtr LoadLibrary(string dllToLoad); @@ -213,5 +160,8 @@ public IntPtr GetFunction(IntPtr hModule, string procedureName) [DllImport(NativeDll)] static extern bool FreeLibrary(IntPtr hModule); + + [DllImport("Psapi.dll", SetLastError = true)] + static extern bool EnumProcessModules(IntPtr hProcess, [In, Out] IntPtr[] lphModule, uint lphModuleByteCount, out uint byteCountNeeded); } } diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs deleted file mode 100644 index b6fe89425..000000000 --- a/src/runtime/platform/NativeCodePage.cs +++ /dev/null @@ -1,226 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace Python.Runtime.Platform -{ - class NativeCodePageHelper - { - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X86: - return I386; - case Architecture.X64: - return X86_64; - default: - return null; - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - return 0x20; - } - else - { - // OSX, FreeBSD - return 0x1000; - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return new WindowsMemoryMapper(); - } - else - { - // Linux, OSX, FreeBSD - return new UnixMemoryMapper(); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } - } -} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs deleted file mode 100644 index 15235da5a..000000000 --- a/src/runtime/platform/Types.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace Python.Runtime.Platform -{ - public enum MachineType - { - i386, - x86_64, - armv7l, - armv8, - aarch64, - Other - }; - - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - - static class SystemInfo - { - public static MachineType GetMachineType() - { - return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); - } - - public static string GetArchitecture() - { - return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); - } - - public static OperatingSystemType GetSystemType() - { - if (Runtime.IsWindows) - { - return OperatingSystemType.Windows; - } - switch (PythonEngine.Platform) - { - case "linux": - return OperatingSystemType.Linux; - - case "darwin": - return OperatingSystemType.Darwin; - - default: - return OperatingSystemType.Other; - } - } - - #region WINDOWS - - static string GetArchName_Windows() - { - // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details - return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); - } - - static MachineType GetMachineType_Windows() - { - if (Runtime.Is32Bit) - { - return MachineType.i386; - } - switch (GetArchName_Windows()) - { - case "AMD64": - return MachineType.x86_64; - case "ARM64": - return MachineType.aarch64; - default: - return MachineType.Other; - } - } - - #endregion - - #region UNIX - - - [StructLayout(LayoutKind.Sequential)] - unsafe struct utsname_linux - { - const int NameLength = 65; - - /* Name of the implementation of the operating system. */ - public fixed byte sysname[NameLength]; - - /* Name of this node on the network. */ - public fixed byte nodename[NameLength]; - - /* Current release level of this implementation. */ - public fixed byte release[NameLength]; - - /* Current version level of this release. */ - public fixed byte version[NameLength]; - - /* Name of the hardware type the system is running on. */ - public fixed byte machine[NameLength]; - - // GNU extension - fixed byte domainname[NameLength]; /* NIS or YP domain name */ - } - - [StructLayout(LayoutKind.Sequential)] - unsafe struct utsname_darwin - { - const int NameLength = 256; - - /* Name of the implementation of the operating system. */ - public fixed byte sysname[NameLength]; - - /* Name of this node on the network. */ - public fixed byte nodename[NameLength]; - - /* Current release level of this implementation. */ - public fixed byte release[NameLength]; - - /* Current version level of this release. */ - public fixed byte version[NameLength]; - - /* Name of the hardware type the system is running on. */ - public fixed byte machine[NameLength]; - } - - [DllImport("libc")] - static extern int uname(IntPtr buf); - - - static unsafe string GetArchName_Unix() - { - switch (GetSystemType()) - { - case OperatingSystemType.Linux: - { - var buf = stackalloc utsname_linux[1]; - if (uname((IntPtr)buf) != 0) - { - return null; - } - return Marshal.PtrToStringAnsi((IntPtr)buf->machine); - } - - case OperatingSystemType.Darwin: - { - var buf = stackalloc utsname_darwin[1]; - if (uname((IntPtr)buf) != 0) - { - return null; - } - return Marshal.PtrToStringAnsi((IntPtr)buf->machine); - } - - default: - return null; - } - } - - static unsafe MachineType GetMachineType_Unix() - { - switch (GetArchName_Unix()) - { - case "x86_64": - case "em64t": - return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; - case "i386": - case "i686": - return MachineType.i386; - - case "armv7l": - return MachineType.armv7l; - case "armv8": - return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; - case "aarch64": - return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; - - default: - return MachineType.Other; - } - } - - #endregion - } -} diff --git a/src/runtime/polyfill/ReflectionPolyfills.cs b/src/runtime/polyfill/ReflectionPolyfills.cs index 65f9b83de..36bd39cef 100644 --- a/src/runtime/polyfill/ReflectionPolyfills.cs +++ b/src/runtime/polyfill/ReflectionPolyfills.cs @@ -30,5 +30,8 @@ public static T GetCustomAttribute(this Assembly assembly) where T: Attribute .Cast() .SingleOrDefault(); } + + public static bool IsFlagsEnum(this Type type) + => type.GetCustomAttribute() is not null; } } diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs index cf657a033..9fe22aca7 100644 --- a/src/runtime/pybuffer.cs +++ b/src/runtime/pybuffer.cs @@ -18,7 +18,7 @@ unsafe internal PyBuffer(PyObject exporter, PyBUF flags) if (Runtime.PyObject_GetBuffer(exporter.Handle, ref _view, (int)flags) < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } _exporter = exporter; @@ -127,7 +127,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) throw new NotSupportedException("FromContiguous requires at least Python 3.7"); if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// @@ -141,7 +141,7 @@ public void ToContiguous(IntPtr buf, BufferOrderStyle order) throw new ObjectDisposedException(nameof(PyBuffer)); if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// @@ -167,7 +167,7 @@ public void FillInfo(IntPtr exporter, IntPtr buf, long len, bool _readonly, int if (disposedValue) throw new ObjectDisposedException(nameof(PyBuffer)); if (Runtime.PyBuffer_FillInfo(ref _view, exporter, buf, (IntPtr)len, Convert.ToInt32(_readonly), flags) < 0) - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } /// diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index ade873f7a..0a5b2ad82 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -22,6 +22,7 @@ public PyDict(IntPtr ptr) : base(ptr) { } + internal PyDict(BorrowedReference reference) : base(reference) { } /// /// PyDict Constructor @@ -33,7 +34,7 @@ public PyDict() : base(Runtime.PyDict_New()) { if (obj == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -103,12 +104,12 @@ public bool HasKey(string key) /// public PyObject Keys() { - IntPtr items = Runtime.PyDict_Keys(obj); - if (items == IntPtr.Zero) + using var items = Runtime.PyDict_Keys(Reference); + if (items.IsNull()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } - return new PyObject(items); + return items.MoveToPyObject(); } @@ -123,7 +124,7 @@ public PyObject Values() IntPtr items = Runtime.PyDict_Values(obj); if (items == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(items); } @@ -137,12 +138,12 @@ public PyObject Values() /// public PyObject Items() { - var items = Runtime.PyDict_Items(this.obj); + var items = Runtime.PyDict_Items(this.Reference); try { if (items.IsNull()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return items.MoveToPyObject(); @@ -165,7 +166,7 @@ public PyDict Copy() IntPtr op = Runtime.PyDict_Copy(obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyDict(op); } @@ -179,10 +180,10 @@ public PyDict Copy() /// public void Update(PyObject other) { - int result = Runtime.PyDict_Update(obj, other.obj); + int result = Runtime.PyDict_Update(Reference, other.Reference); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } diff --git a/src/runtime/pyfloat.cs b/src/runtime/pyfloat.cs index b07b95de1..a1752ff68 100644 --- a/src/runtime/pyfloat.cs +++ b/src/runtime/pyfloat.cs @@ -66,9 +66,9 @@ private static IntPtr FromString(string value) { using (var s = new PyString(value)) { - IntPtr val = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); + NewReference val = Runtime.PyFloat_FromString(s.Reference); PythonException.ThrowIfIsNull(val); - return val; + return val.DangerousMoveToPointerOrNull(); } } diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f8718cb95..7b02c68e5 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -149,7 +149,7 @@ public PyInt(sbyte value) : this((int)value) private static IntPtr FromString(string value) { - IntPtr val = Runtime.PyInt_FromString(value, IntPtr.Zero, 0); + IntPtr val = Runtime.PyLong_FromString(value, IntPtr.Zero, 0); PythonException.ThrowIfIsNull(val); return val; } diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index 2016ef4f8..da2a600c6 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -57,7 +57,7 @@ public bool MoveNext() { if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } // stop holding the previous object, if there was one diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 7f5566401..8f346524f 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -61,7 +61,7 @@ public PyList() : base(Runtime.PyList_New(0)) { if (obj == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -77,7 +77,7 @@ private static IntPtr FromArray(PyObject[] items) if (r < 0) { Runtime.Py_DecRef(val); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } return val; @@ -118,7 +118,7 @@ public static PyList AsList(PyObject value) IntPtr op = Runtime.PySequence_List(value.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyList(op); } @@ -132,10 +132,10 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(this.Reference, item.obj); + int r = Runtime.PyList_Append(this.Reference, new BorrowedReference(item.obj)); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -150,7 +150,7 @@ public void Insert(int index, PyObject item) int r = Runtime.PyList_Insert(this.Reference, index, item.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -166,7 +166,7 @@ public void Reverse() int r = Runtime.PyList_Reverse(this.Reference); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -182,7 +182,7 @@ public void Sort() int r = Runtime.PyList_Sort(this.Reference); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } diff --git a/src/runtime/pylong.cs b/src/runtime/pylong.cs index 0e21c7c49..8cb814cf6 100644 --- a/src/runtime/pylong.cs +++ b/src/runtime/pylong.cs @@ -188,11 +188,8 @@ public PyLong(string value) : base(FromString(value)) /// - /// IsLongType Method - /// - /// /// Returns true if the given object is a Python long. - /// + /// public static bool IsLongType(PyObject value) { return Runtime.PyLong_Check(value.obj); @@ -246,7 +243,7 @@ public int ToInt32() /// public long ToInt64() { - return Runtime.PyLong_AsLongLong(obj); + return Runtime.PyExplicitlyConvertToInt64(obj); } } } diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs new file mode 100644 index 000000000..800edb686 --- /dev/null +++ b/src/runtime/pymodule.cs @@ -0,0 +1,41 @@ +using System; + +namespace Python.Runtime +{ + public class PyModule : PyScope + { + internal PyModule(ref NewReference reference) : base(ref reference, PyScopeManager.Global) { } + public PyModule(PyObject o) : base(o.Reference, PyScopeManager.Global) { } + + /// + /// Given a module or package name, import the + /// module and return the resulting module object as a . + /// + /// Fully-qualified module or package name + public static PyModule Import(string name) + { + NewReference op = Runtime.PyImport_ImportModule(name); + PythonException.ThrowIfIsNull(op); + return new PyModule(ref op); + } + + /// + /// Reloads the module, and returns the updated object + /// + public PyModule Reload() + { + NewReference op = Runtime.PyImport_ReloadModule(this.Reference); + PythonException.ThrowIfIsNull(op); + return new PyModule(ref op); + } + + public static PyModule FromString(string name, string code) + { + using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); + PythonException.ThrowIfIsNull(c); + NewReference m = Runtime.PyImport_ExecCodeModule(name, c); + PythonException.ThrowIfIsNull(m); + return new PyModule(ref m); + } + } +} diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index d68a9905b..dea12ba1b 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -27,7 +27,8 @@ public partial class PyObject : DynamicObject, IEnumerable, IDisposabl protected internal IntPtr obj = IntPtr.Zero; - internal BorrowedReference Reference => new BorrowedReference(obj); + public static PyObject None => new PyObject(new BorrowedReference(Runtime.PyNone)); + internal BorrowedReference Reference => new BorrowedReference(this.obj); /// /// PyObject Constructor @@ -78,6 +79,17 @@ internal PyObject(BorrowedReference reference) #endif } + internal PyObject(StolenReference reference) + { + if (reference == null) throw new ArgumentNullException(nameof(reference)); + + obj = reference.DangerousGetAddressOrNull(); + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif + } + // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. ~PyObject() @@ -123,6 +135,13 @@ public static PyObject FromManagedObject(object ob) return new PyObject(op); } + /// + /// Creates new from a nullable reference. + /// When is null, null is returned. + /// + internal static PyObject FromNullableReference(BorrowedReference reference) + => reference.IsNull ? null : new PyObject(reference); + /// /// AsManagedObject Method @@ -136,7 +155,8 @@ public object AsManagedObject(Type t) object result; if (!Converter.ToManaged(obj, t, out result, true)) { - throw new InvalidCastException("cannot convert object to target type", new PythonException()); + throw new InvalidCastException("cannot convert object to target type", + PythonException.FetchCurrentOrNull(out _)); } return result; } @@ -157,6 +177,7 @@ public T As() return (T)AsManagedObject(typeof(T)); } + internal bool IsDisposed => obj == IntPtr.Zero; /// /// Dispose Method @@ -197,7 +218,7 @@ protected virtual void Dispose(bool disposing) { // Python requires finalizers to preserve exception: // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation - Runtime.PyErr_Restore(errType, errVal, traceback); + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); } } else @@ -218,6 +239,9 @@ public void Dispose() GC.SuppressFinalize(this); } + internal BorrowedReference GetPythonTypeReference() + => new BorrowedReference(Runtime.PyObject_TYPE(obj)); + /// /// GetPythonType Method /// @@ -257,7 +281,7 @@ public bool HasAttr(string name) { if (name == null) throw new ArgumentNullException(nameof(name)); - return Runtime.PyObject_HasAttrString(obj, name) != 0; + return Runtime.PyObject_HasAttrString(Reference, name) != 0; } @@ -272,7 +296,7 @@ public bool HasAttr(PyObject name) { if (name == null) throw new ArgumentNullException(nameof(name)); - return Runtime.PyObject_HasAttr(obj, name.obj) != 0; + return Runtime.PyObject_HasAttr(Reference, name.Reference) != 0; } @@ -290,7 +314,7 @@ public PyObject GetAttr(string name) IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -332,7 +356,7 @@ public PyObject GetAttr(PyObject name) IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -375,7 +399,7 @@ public void SetAttr(string name, PyObject value) int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -396,7 +420,7 @@ public void SetAttr(PyObject name, PyObject value) int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -415,7 +439,7 @@ public void DelAttr(string name) int r = Runtime.PyObject_SetAttrString(obj, name, IntPtr.Zero); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -435,7 +459,7 @@ public void DelAttr(PyObject name) int r = Runtime.PyObject_SetAttr(obj, name.obj, IntPtr.Zero); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -455,7 +479,7 @@ public virtual PyObject GetItem(PyObject key) IntPtr op = Runtime.PyObject_GetItem(obj, key.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -513,7 +537,7 @@ public virtual void SetItem(PyObject key, PyObject value) int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -572,7 +596,7 @@ public virtual void DelItem(PyObject key) int r = Runtime.PyObject_DelItem(obj, key.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -614,19 +638,15 @@ public virtual void DelItem(int index) /// - /// Length Method - /// - /// /// Returns the length for objects that support the Python sequence - /// protocol, or 0 if the object does not support the protocol. - /// + /// protocol. + /// public virtual long Length() { - var s = Runtime.PyObject_Size(obj); + var s = Runtime.PyObject_Size(Reference); if (s < 0) { - Runtime.PyErr_Clear(); - return 0; + throw PythonException.ThrowLastAsClrException(); } return s; } @@ -687,7 +707,7 @@ public PyObject GetIterator() IntPtr r = Runtime.PyObject_GetIter(obj); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -724,7 +744,7 @@ public PyObject Invoke(params PyObject[] args) t.Dispose(); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -744,7 +764,7 @@ public PyObject Invoke(PyTuple args) IntPtr r = Runtime.PyObject_Call(obj, args.obj, IntPtr.Zero); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -767,7 +787,7 @@ public PyObject Invoke(PyObject[] args, PyDict kw) t.Dispose(); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -787,7 +807,7 @@ public PyObject Invoke(PyTuple args, PyDict kw) IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw?.obj ?? IntPtr.Zero); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(r); } @@ -933,17 +953,21 @@ public bool IsInstance(PyObject typeOrClass) /// - /// IsSubclass Method - /// - /// - /// Return true if the object is identical to or derived from the + /// Return true if the object is identical to or derived from the /// given Python type or class. This method always succeeds. - /// + /// public bool IsSubclass(PyObject typeOrClass) { if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); - int r = Runtime.PyObject_IsSubclass(obj, typeOrClass.obj); + return IsSubclass(typeOrClass.Reference); + } + + internal bool IsSubclass(BorrowedReference typeOrClass) + { + if (typeOrClass.IsNull) throw new ArgumentNullException(nameof(typeOrClass)); + + int r = Runtime.PyObject_IsSubclass(Reference, typeOrClass); if (r < 0) { Runtime.PyErr_Clear(); @@ -1008,7 +1032,7 @@ public PyList Dir() IntPtr r = Runtime.PyObject_Dir(obj); if (r == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyList(r); } @@ -1039,7 +1063,7 @@ public string Repr() /// public override string ToString() { - IntPtr strval = Runtime.PyObject_Unicode(obj); + IntPtr strval = Runtime.PyObject_Str(obj); string result = Runtime.GetManagedString(strval); Runtime.XDecref(strval); return result; @@ -1066,7 +1090,7 @@ public override bool Equals(object o) int r = Runtime.PyObject_Compare(obj, ((PyObject)o).obj); if (Exceptions.ErrorOccurred()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return r == 0; } @@ -1120,7 +1144,7 @@ public override bool TrySetMember(SetMemberBinder binder, object value) int r = Runtime.PyObject_SetAttrString(obj, binder.Name, ptr); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } Runtime.XDecref(ptr); return true; @@ -1192,7 +1216,7 @@ private static void AddArgument(IntPtr argtuple, int i, object target) if (Runtime.PyTuple_SetItem(argtuple, i, ptr) < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -1440,4 +1464,13 @@ public override IEnumerable GetDynamicMemberNames() } } } + + internal static class PyObjectExtensions + { + internal static NewReference NewReferenceOrNull(this PyObject self) + => NewReference.DangerousFromPointer( + (self?.obj ?? IntPtr.Zero) == IntPtr.Zero + ? IntPtr.Zero + : Runtime.SelfIncRef(self.obj)); + } } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index d61573733..e1b499c5c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,22 +22,15 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : PyObject { - public readonly string Name; + public string Name { get; } /// - /// the python Module object the scope associated with. - /// - internal IntPtr obj; - - /// - /// the variable dict of the scope. + /// the variable dict of the scope. Borrowed. /// internal readonly IntPtr variables; - - private bool _isDisposed; - private bool _finalized = false; + internal BorrowedReference VarsRef => new BorrowedReference(variables); /// /// The Manager this scope associated with. @@ -50,36 +43,40 @@ public class PyScope : DynamicObject, IDisposable /// public event Action OnDispose; - /// - /// Constructor - /// - /// - /// Create a scope based on a Python Module. - /// - internal PyScope(IntPtr ptr, PyScopeManager manager) + /// Create a scope based on a Python Module. + internal PyScope(ref NewReference reference, PyScopeManager manager) + : this(reference.DangerousMoveToPointer(), manager) { } + /// Create a scope based on a Python Module. + internal PyScope(BorrowedReference reference, PyScopeManager manager) + : this(reference.DangerousGetAddress(), manager) + { + Runtime.XIncref(reference.DangerousGetAddress()); + } + + /// Create a scope based on a Python Module. + private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr) { - if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(ptr), Runtime.PyModuleType)) + if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(Reference), Runtime.PyModuleType)) { throw new PyScopeException("object is not a module"); } Manager = manager ?? PyScopeManager.Global; - obj = ptr; //Refcount of the variables not increase - variables = Runtime.PyModule_GetDict(obj); + variables = Runtime.PyModule_GetDict(Reference).DangerousGetAddress(); PythonException.ThrowIfIsNull(variables); int res = Runtime.PyDict_SetItem( - variables, PyIdentifier.__builtins__, + VarsRef, PyIdentifier.__builtins__, Runtime.PyEval_GetBuiltins() ); PythonException.ThrowIfIsNotZero(res); - this.Name = this.Get("__name__"); + using var name = this.Get("__name__"); + this.Name = name.As(); } /// /// return the variable dict of the scope. /// - /// public PyDict Variables() { Runtime.XIncref(variables); @@ -120,7 +117,7 @@ public dynamic Import(string name, string asname = null) } else { - PyObject module = PythonEngine.ImportModule(name); + var module = PyModule.Import(name); Import(module, asname); return module; } @@ -134,7 +131,7 @@ public dynamic Import(string name, string asname = null) /// public void Import(PyScope scope, string asname) { - this.Set(asname, scope.obj); + this.SetPyValue(asname, scope.Handle); } /// @@ -171,7 +168,7 @@ public void ImportAll(string name) } else { - PyObject module = PythonEngine.ImportModule(name); + var module = PyModule.Import(name); ImportAll(module); } } @@ -184,10 +181,10 @@ public void ImportAll(string name) /// public void ImportAll(PyScope scope) { - int result = Runtime.PyDict_Update(variables, scope.variables); + int result = Runtime.PyDict_Update(VarsRef, scope.VarsRef); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -203,11 +200,11 @@ public void ImportAll(PyObject module) { throw new PyScopeException("object is not a module"); } - var module_dict = Runtime.PyModule_GetDict(module.obj); - int result = Runtime.PyDict_Update(variables, module_dict); + var module_dict = Runtime.PyModule_GetDict(module.Reference); + int result = Runtime.PyDict_Update(VarsRef, module_dict); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -219,10 +216,10 @@ public void ImportAll(PyObject module) /// public void ImportAll(PyDict dict) { - int result = Runtime.PyDict_Update(variables, dict.obj); + int result = Runtime.PyDict_Update(VarsRef, dict.Reference); if (result < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -277,10 +274,10 @@ public T Execute(PyObject script, PyDict locals = null) public PyObject Eval(string code, PyDict locals = null) { Check(); - IntPtr _locals = locals == null ? variables : locals.obj; + BorrowedReference _locals = locals == null ? VarsRef : locals.Reference; NewReference reference = Runtime.PyRun_String( - code, RunFlagType.Eval, variables, _locals + code, RunFlagType.Eval, VarsRef, _locals ); PythonException.ThrowIfIsNull(reference); return reference.MoveToPyObject(); @@ -310,17 +307,16 @@ public T Eval(string code, PyDict locals = null) public void Exec(string code, PyDict locals = null) { Check(); - IntPtr _locals = locals == null ? variables : locals.obj; - Exec(code, variables, _locals); + BorrowedReference _locals = locals == null ? VarsRef : locals.Reference; + Exec(code, VarsRef, _locals); } - private void Exec(string code, IntPtr _globals, IntPtr _locals) + private void Exec(string code, BorrowedReference _globals, BorrowedReference _locals) { - NewReference reference = Runtime.PyRun_String( + using NewReference reference = Runtime.PyRun_String( code, RunFlagType.File, _globals, _locals ); PythonException.ThrowIfIsNull(reference); - reference.Dispose(); } /// @@ -333,11 +329,11 @@ private void Exec(string code, IntPtr _globals, IntPtr _locals) public void Set(string name, object value) { IntPtr _value = Converter.ToPython(value, value?.GetType()); - Set(name, _value); + SetPyValue(name, _value); Runtime.XDecref(_value); } - private void Set(string name, IntPtr value) + private void SetPyValue(string name, IntPtr value) { Check(); using (var pyKey = new PyString(name)) @@ -345,7 +341,7 @@ private void Set(string name, IntPtr value) int r = Runtime.PyObject_SetItem(variables, pyKey.obj, value); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -364,7 +360,7 @@ public void Remove(string name) int r = Runtime.PyObject_DelItem(variables, pyKey.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } } @@ -419,7 +415,7 @@ public bool TryGet(string name, out PyObject value) IntPtr op = Runtime.PyObject_GetItem(variables, pyKey.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } if (op == Runtime.PyNone) { @@ -505,32 +501,21 @@ public override bool TrySetMember(SetMemberBinder binder, object value) private void Check() { - if (_isDisposed) + if (this.obj == IntPtr.Zero) { throw new PyScopeException($"The scope of name '{Name}' object has been disposed"); } } - public void Dispose() + protected override void Dispose(bool disposing) { - if (_isDisposed) + if (this.obj == IntPtr.Zero) { return; } - _isDisposed = true; - Runtime.XDecref(obj); + base.Dispose(disposing); this.OnDispose?.Invoke(this); } - - ~PyScope() - { - if (_finalized || _isDisposed) - { - return; - } - _finalized = true; - Finalizer.Instance.AddFinalizedObject(ref obj); - } } public class PyScopeManager @@ -551,11 +536,11 @@ internal PyScope NewScope(string name) name = ""; } var module = Runtime.PyModule_New(name); - if (module == IntPtr.Zero) + if (module.IsNull()) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } - return new PyScope(module, this); + return new PyScope(ref module, this); } /// diff --git a/src/runtime/pysequence.cs b/src/runtime/pysequence.cs index 1850ef7de..536cd3e46 100644 --- a/src/runtime/pysequence.cs +++ b/src/runtime/pysequence.cs @@ -42,7 +42,7 @@ public PyObject GetSlice(int i1, int i2) IntPtr op = Runtime.PySequence_GetSlice(obj, i1, i2); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -59,7 +59,7 @@ public void SetSlice(int i1, int i2, PyObject v) int r = Runtime.PySequence_SetSlice(obj, i1, i2, v.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -75,7 +75,7 @@ public void DelSlice(int i1, int i2) int r = Runtime.PySequence_DelSlice(obj, i1, i2); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -111,7 +111,7 @@ public bool Contains(PyObject item) int r = Runtime.PySequence_Contains(obj, item.obj); if (r < 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return r != 0; } @@ -129,7 +129,7 @@ public PyObject Concat(PyObject other) IntPtr op = Runtime.PySequence_Concat(obj, other.obj); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } @@ -147,7 +147,7 @@ public PyObject Repeat(int count) IntPtr op = Runtime.PySequence_Repeat(obj, count); if (op == IntPtr.Zero) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } return new PyObject(op); } diff --git a/src/runtime/pystring.cs b/src/runtime/pystring.cs index b3d0dc86d..172c09ebd 100644 --- a/src/runtime/pystring.cs +++ b/src/runtime/pystring.cs @@ -51,7 +51,7 @@ public PyString(PyObject o) : base(FromObject(o)) private static IntPtr FromString(string s) { - IntPtr val = Runtime.PyUnicode_FromUnicode(s, s.Length); + IntPtr val = Runtime.PyString_FromString(s); PythonException.ThrowIfIsNull(val); return val; } @@ -67,11 +67,8 @@ public PyString(string s) : base(FromString(s)) /// - /// IsStringType Method - /// - /// /// Returns true if the given object is a Python string. - /// + /// public static bool IsStringType(PyObject value) { return Runtime.PyString_Check(value.obj); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index df6cf7101..f6340a59c 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading; namespace Python.Runtime { @@ -51,6 +52,9 @@ public static bool IsInitialized get { return initialized; } } + /// Set to true to enable GIL debugging assistance. + public static bool DebugGIL { get; set; } = false; + internal static DelegateManager DelegateManager { get @@ -188,6 +192,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // Make sure we clean up properly on app domain unload. AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; // The global scope gets used implicitly quite early on, remember // to clear it out when we shut down. @@ -198,28 +203,16 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Py.SetArgv(args); } - if (mode == ShutdownMode.Normal) - { - // TOOD: Check if this can be remove completely or not. - // register the atexit callback (this doesn't use Py_AtExit as the C atexit - // callbacks are called after python is fully finalized but the python ones - // are called while the python engine is still running). - //string code = - // "import atexit, clr\n" + - // "atexit.register(clr._AtExit)\n"; - //PythonEngine.Exec(code); - } - // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + NewReference clr = Python.Runtime.ImportHook.GetCLRModule(); + BorrowedReference clr_dict = Runtime.PyModule_GetDict(clr); var locals = new PyDict(); try { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); + BorrowedReference module = Runtime.PyImport_AddModule("clr._extras"); + BorrowedReference module_globals = Runtime.PyModule_GetDict(module); + BorrowedReference builtins = Runtime.PyEval_GetBuiltins(); Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); @@ -228,7 +221,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, { // add the contents of clr.py to the module string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); + Exec(clr_py, module_globals, locals.Reference); } // add the imported module to the clr module, and copy the API functions @@ -240,7 +233,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + Runtime.PyDict_SetItem(clr_dict, key.Reference, value.Reference); value.Dispose(); } key.Dispose(); @@ -257,6 +250,11 @@ static void OnDomainUnload(object _, EventArgs __) Shutdown(); } + static void OnProcessExit(object _, EventArgs __) + { + Shutdown(); + } + /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -266,7 +264,7 @@ public static IntPtr InitExt() { try { - Initialize(setSysArgv: false); + Initialize(setSysArgv: false, mode: ShutdownMode.Extension); // Trickery - when the import hook is installed into an already // running Python, the standard import machinery is still in @@ -305,7 +303,8 @@ public static IntPtr InitExt() return IntPtr.Zero; } - return Python.Runtime.ImportHook.GetCLRModule(); + return Python.Runtime.ImportHook.GetCLRModule() + .DangerousMoveToPointerOrNull(); } /// @@ -326,6 +325,7 @@ public static void Shutdown(ShutdownMode mode) // If the shutdown handlers trigger a domain unload, // don't call shutdown again. AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; PyScopeManager.Global.Clear(); ExecuteShutdownHandlers(); @@ -479,62 +479,27 @@ public static void EndAllowThreads(IntPtr ts) Runtime.PyEval_RestoreThread(ts); } + [Obsolete("Use PyModule.Import")] + public static PyObject ImportModule(string name) => PyModule.Import(name); - /// - /// ImportModule Method - /// - /// - /// Given a fully-qualified module or package name, import the - /// module and return the resulting module object as a PyObject - /// or null if an exception is raised. - /// - public static PyObject ImportModule(string name) - { - IntPtr op = Runtime.PyImport_ImportModule(name); - PythonException.ThrowIfIsNull(op); - return new PyObject(op); - } - - - /// - /// ReloadModule Method - /// - /// - /// Given a PyObject representing a previously loaded module, reload - /// the module. - /// + [Obsolete("Use PyModule.Reload")] public static PyObject ReloadModule(PyObject module) - { - IntPtr op = Runtime.PyImport_ReloadModule(module.Handle); - PythonException.ThrowIfIsNull(op); - return new PyObject(op); - } + => module is PyModule pyModule ? pyModule.Reload() : new PyModule(module).Reload(); - - /// - /// ModuleFromString Method - /// - /// - /// Given a string module name and a string containing Python code, - /// execute the code in and return a module of the given name. - /// + [Obsolete("Use PyModule.FromString")] public static PyObject ModuleFromString(string name, string code) - { - IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); - PythonException.ThrowIfIsNull(c); - IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); - PythonException.ThrowIfIsNull(m); - return new PyObject(m); - } + => PyModule.FromString(name, code); + public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) { var flag = (int)mode; - IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); + NewReference ptr = Runtime.Py_CompileString(code, filename, flag); PythonException.ThrowIfIsNull(ptr); - return new PyObject(ptr); + return ptr.MoveToPyObject(); } + /// /// Eval Method /// @@ -544,7 +509,9 @@ public static PyObject Compile(string code, string filename = "", RunFlagType mo /// public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals = null) { - PyObject result = RunString(code, globals, locals, RunFlagType.Eval); + var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); + var localsRef = new BorrowedReference(locals.GetValueOrDefault()); + PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.Eval); return result; } @@ -558,15 +525,54 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) { - using (PyObject result = RunString(code, globals, locals, RunFlagType.File)) + var globalsRef = new BorrowedReference(globals.GetValueOrDefault()); + var localsRef = new BorrowedReference(locals.GetValueOrDefault()); + using PyObject result = RunString(code, globalsRef, localsRef, RunFlagType.File); + if (result.obj != Runtime.PyNone) { - if (result.obj != Runtime.PyNone) - { - throw new PythonException(); - } + throw PythonException.ThrowLastAsClrException(); } } + /// + /// Exec Method + /// + /// + /// Run a string containing Python code. + /// It's a subset of Python exec function. + /// + internal static void Exec(string code, BorrowedReference globals, BorrowedReference locals = default) + { + using PyObject result = RunString(code, globals: globals, locals: locals, RunFlagType.File); + if (result.obj != Runtime.PyNone) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + /// + /// Gets the Python thread ID. + /// + /// The Python thread ID. + public static ulong GetPythonThreadID() + { + dynamic threading = Py.Import("threading"); + return threading.InvokeMethod("get_ident"); + } + + /// + /// Interrupts the execution of a thread. + /// + /// The Python thread ID. + /// The number of thread states modified; this is normally one, but will be zero if the thread id isn’t found. + public static int Interrupt(ulong pythonThreadID) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Runtime.PyThreadState_SetAsyncExcLLP64((uint)pythonThreadID, Exceptions.KeyboardInterrupt); + } + return Runtime.PyThreadState_SetAsyncExcLP64(pythonThreadID, Exceptions.KeyboardInterrupt); + } /// /// RunString Method. Function has been deprecated and will be removed. @@ -575,7 +581,7 @@ public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = nu [Obsolete("RunString is deprecated and will be removed. Use Exec/Eval/RunSimpleString instead.")] public static PyObject RunString(string code, IntPtr? globals = null, IntPtr? locals = null) { - return RunString(code, globals, locals, RunFlagType.File); + return RunString(code, new BorrowedReference(globals.GetValueOrDefault()), new BorrowedReference(locals.GetValueOrDefault()), RunFlagType.File); } /// @@ -586,20 +592,19 @@ public static PyObject RunString(string code, IntPtr? globals = null, IntPtr? lo /// executing the code string as a PyObject instance, or null if /// an exception was raised. /// - internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, RunFlagType flag) + internal static PyObject RunString(string code, BorrowedReference globals, BorrowedReference locals, RunFlagType flag) { - var borrowedGlobals = true; - if (globals == null) + NewReference tempGlobals = default; + if (globals.IsNull) { globals = Runtime.PyEval_GetGlobals(); - if (globals == IntPtr.Zero) + if (globals.IsNull) { - globals = Runtime.PyDict_New(); + globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New()); Runtime.PyDict_SetItem( - globals.Value, PyIdentifier.__builtins__, + globals, PyIdentifier.__builtins__, Runtime.PyEval_GetBuiltins() ); - borrowedGlobals = false; } } @@ -611,17 +616,14 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { NewReference result = Runtime.PyRun_String( - code, flag, globals.Value, locals.Value + code, flag, globals, locals ); PythonException.ThrowIfIsNull(result); return result.MoveToPyObject(); } finally { - if (!borrowedGlobals) - { - Runtime.XDecref(globals.Value); - } + tempGlobals.Dispose(); } } } @@ -642,7 +644,7 @@ public static GILState GIL() PythonEngine.Initialize(); } - return new GILState(); + return PythonEngine.DebugGIL ? new DebugGILState() : new GILState(); } public static PyScope CreateScope() @@ -667,7 +669,7 @@ internal GILState() state = PythonEngine.AcquireLock(); } - public void Dispose() + public virtual void Dispose() { if (this.isDisposed) return; @@ -678,7 +680,23 @@ public void Dispose() ~GILState() { - Dispose(); + throw new InvalidOperationException("GIL must always be released, and it must be released from the same thread that acquired it."); + } + } + + public class DebugGILState : GILState + { + readonly Thread owner; + internal DebugGILState() : base() + { + this.owner = Thread.CurrentThread; + } + public override void Dispose() + { + if (this.owner != Thread.CurrentThread) + throw new InvalidOperationException("GIL must always be released from the same thread, that acquired it"); + + base.Dispose(); } } @@ -716,10 +734,12 @@ public static KeywordArguments kw(params object[] kv) return dict; } - public static PyObject Import(string name) - { - return PythonEngine.ImportModule(name); - } + /// + /// Given a module or package name, import the + /// module and return the resulting module object as a . + /// + /// Fully-qualified module or package name + public static PyModule Import(string name) => PyModule.Import(name); public static void SetArgv() { @@ -760,10 +780,8 @@ public static void With(PyObject obj, Action Body) // Behavior described here: // https://docs.python.org/2/reference/datamodel.html#with-statement-context-managers - IntPtr type = Runtime.PyNone; - IntPtr val = Runtime.PyNone; - IntPtr traceBack = Runtime.PyNone; - PythonException ex = null; + Exception ex = null; + PythonException pyError = null; try { @@ -772,17 +790,21 @@ public static void With(PyObject obj, Action Body) Body(enterResult); } catch (PythonException e) + { + ex = pyError = e; + } + catch (Exception e) { ex = e; - type = ex.PyType.Coalesce(type); - val = ex.PyValue.Coalesce(val); - traceBack = ex.PyTB.Coalesce(traceBack); + Exceptions.SetError(e); + pyError = PythonException.FetchCurrentRaw(); } - Runtime.XIncref(type); - Runtime.XIncref(val); - Runtime.XIncref(traceBack); - var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); + PyObject type = pyError?.Type ?? PyObject.None; + PyObject val = pyError?.Value ?? PyObject.None; + PyObject traceBack = pyError?.Traceback ?? PyObject.None; + + var exitResult = obj.InvokeMethod("__exit__", type, val, traceBack); if (ex != null && !exitResult.IsTrue()) throw ex; } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index eff33d699..cca7c439f 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,5 +1,9 @@ +#nullable enable using System; +using System.Diagnostics; +using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Text; namespace Python.Runtime @@ -8,126 +12,236 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception, IDisposable + public class PythonException : System.Exception { - private IntPtr _pyType = IntPtr.Zero; - private IntPtr _pyValue = IntPtr.Zero; - private IntPtr _pyTB = IntPtr.Zero; - private string _tb = ""; - private string _message = ""; - private string _pythonTypeName = ""; - private bool disposed = false; - private bool _finalized = false; - - public PythonException() + public PythonException(PyType type, PyObject? value, PyObject? traceback, + string message, Exception? innerException) + : base(message, innerException) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB); - if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + Type = type ?? throw new ArgumentNullException(nameof(type)); + Value = value; + Traceback = traceback; + } + + public PythonException(PyType type, PyObject? value, PyObject? traceback, + Exception? innerException) + : this(type, value, traceback, GetMessage(value, type), innerException) { } + + public PythonException(PyType type, PyObject? value, PyObject? traceback) + : this(type, value, traceback, innerException: null) { } + + /// + /// Rethrows the last Python exception as corresponding CLR exception. + /// It is recommended to call this as throw ThrowLastAsClrException() + /// to assist control flow checks. + /// + internal static Exception ThrowLastAsClrException() + { + var exception = FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) + ?? throw new InvalidOperationException("No exception is set"); + dispatchInfo?.Throw(); + // when dispatchInfo is not null, this line will not be reached + throw exception; + } + + internal static PythonException? FetchCurrentOrNullRaw() + { + using var _ = new Py.GILState(); + + Runtime.PyErr_Fetch(type: out var type, val: out var value, tb: out var traceback); + + if (type.IsNull()) { - string type; - string message; - Runtime.XIncref(_pyType); - using (var pyType = new PyObject(_pyType)) - using (PyObject pyTypeName = pyType.GetAttr("__name__")) - { - type = pyTypeName.ToString(); - } + Debug.Assert(value.IsNull()); + Debug.Assert(traceback.IsNull()); + return null; + } + + return new PythonException( + type: new PyType(type.Steal()), + value: value.MoveToPyObjectOrNull(), + traceback: traceback.MoveToPyObjectOrNull()); + } + internal static PythonException FetchCurrentRaw() + => FetchCurrentOrNullRaw() + ?? throw new InvalidOperationException("No exception is set"); + + internal static Exception? FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) + { + dispatchInfo = null; + + // prevent potential interop errors in this method + // from crashing process with undebuggable StackOverflowException + RuntimeHelpers.EnsureSufficientExecutionStack(); + + using var _ = new Py.GILState(); + Runtime.PyErr_Fetch(out var type, out var value, out var traceback); + if (type.IsNull()) + { + Debug.Assert(value.IsNull()); + Debug.Assert(traceback.IsNull()); + return null; + } - _pythonTypeName = type; + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref traceback); - Runtime.XIncref(_pyValue); - using (var pyValue = new PyObject(_pyValue)) + try + { + return FromPyErr(typeRef: type, valRef: value, tbRef: traceback, out dispatchInfo); + } + finally + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + } + } + + internal static Exception FetchCurrent() + => FetchCurrentOrNull(out _) + ?? throw new InvalidOperationException("No exception is set"); + + private static ExceptionDispatchInfo? TryGetDispatchInfo(BorrowedReference exception) + { + if (exception.IsNull) return null; + + var pyInfo = Runtime.PyObject_GetAttrString(exception, Exceptions.DispatchInfoAttribute); + if (pyInfo.IsNull()) + { + if (Exceptions.ExceptionMatches(Exceptions.AttributeError)) { - message = pyValue.ToString(); + Exceptions.Clear(); } - _message = type + " : " + message; + return null; } - if (_pyTB != IntPtr.Zero) + + try { - using (PyObject tb_module = PythonEngine.ImportModule("traceback")) + if (Converter.ToManagedValue(pyInfo, typeof(ExceptionDispatchInfo), out object result, setError: false)) { - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) - { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); - } + return (ExceptionDispatchInfo)result; } + + return null; + } + finally + { + pyInfo.Dispose(); } - PythonEngine.ReleaseLock(gs); } - // Ensure that encapsulated Python objects are decref'ed appropriately - // when the managed exception wrapper is garbage-collected. - - ~PythonException() + /// + /// Requires lock to be acquired elsewhere + /// + private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef, + out ExceptionDispatchInfo? exceptionDispatchInfo) { - if (_finalized || disposed) + if (valRef == null) throw new ArgumentNullException(nameof(valRef)); + + var type = PyType.FromReference(typeRef); + var value = new PyObject(valRef); + var traceback = PyObject.FromNullableReference(tbRef); + + exceptionDispatchInfo = TryGetDispatchInfo(valRef); + if (exceptionDispatchInfo != null) { - return; + return exceptionDispatchInfo.SourceException; } - _finalized = true; - Finalizer.Instance.AddFinalizedObject(ref _pyType); - Finalizer.Instance.AddFinalizedObject(ref _pyValue); - Finalizer.Instance.AddFinalizedObject(ref _pyTB); + + if (ManagedType.GetManagedObject(valRef) is CLRObject { inst: Exception e }) + { + return e; + } + + if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object decoded) + && decoded is Exception decodedException) + { + return decodedException; + } + + using var cause = Runtime.PyException_GetCause(valRef); + Exception? inner = FromCause(cause); + return new PythonException(type, value, traceback, inner); } - /// - /// Restores python error. - /// - public void Restore() + private static Exception? FromCause(BorrowedReference cause) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Restore(_pyType, _pyValue, _pyTB); - _pyType = IntPtr.Zero; - _pyValue = IntPtr.Zero; - _pyTB = IntPtr.Zero; - PythonEngine.ReleaseLock(gs); + if (cause == null || cause.IsNone()) return null; + + Debug.Assert(Runtime.PyObject_TypeCheck(cause, new BorrowedReference(Exceptions.BaseException))); + + using var innerTraceback = Runtime.PyException_GetTraceback(cause); + return FromPyErr( + typeRef: Runtime.PyObject_TYPE(cause), + valRef: cause, + tbRef: innerTraceback, + out _); + } - /// - /// PyType Property - /// - /// - /// Returns the exception type as a Python object. - /// - public IntPtr PyType + private static string GetMessage(PyObject? value, PyType type) { - get { return _pyType; } + if (type is null) throw new ArgumentNullException(nameof(type)); + + if (value != null && !value.IsNone()) + { + return value.ToString(); + } + + return type.Name; } - /// - /// PyValue Property - /// - /// - /// Returns the exception value as a Python object. - /// - public IntPtr PyValue + private static string TracebackToString(PyObject traceback) { - get { return _pyValue; } + if (traceback is null) + { + throw new ArgumentNullException(nameof(traceback)); + } + + using var tracebackModule = PyModule.Import("traceback"); + using var stackLines = new PyList(tracebackModule.InvokeMethod("format_tb", traceback)); + stackLines.Reverse(); + var result = new StringBuilder(); + foreach (PyObject stackLine in stackLines) + { + result.Append(stackLine); + stackLine.Dispose(); + } + return result.ToString(); } - /// - /// PyTB Property - /// - /// - /// Returns the TraceBack as a Python object. - /// - public IntPtr PyTB + /// Restores python error. + public void Restore() { - get { return _pyTB; } + CheckRuntimeIsRunning(); + + using var _ = new Py.GILState(); + + NewReference type = Type.NewReferenceOrNull(); + NewReference value = Value.NewReferenceOrNull(); + NewReference traceback = Traceback.NewReferenceOrNull(); + + Runtime.PyErr_Restore( + type: type.Steal(), + val: value.StealNullable(), + tb: traceback.StealNullable()); } /// - /// Message Property + /// Returns the exception type as a Python object. + /// + public PyType Type { get; private set; } + + /// + /// Returns the exception value as a Python object. /// + /// + public PyObject? Value { get; private set; } + /// - /// A string representing the python exception message. + /// Returns the TraceBack as a Python object. /// - public override string Message - { - get { return _message; } - } + public PyObject? Traceback { get; } /// /// StackTrace Property @@ -137,146 +251,153 @@ public override string Message /// public override string StackTrace { - get { return _tb + base.StackTrace; } + get + { + if (Traceback is null) return base.StackTrace; + + if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) + return "Python stack unavailable as runtime was shut down\n" + base.StackTrace; + + using var _ = new Py.GILState(); + return TracebackToString(Traceback) + base.StackTrace; + } } - /// - /// Python error type name. - /// - public string PythonTypeName + public bool IsNormalized { - get { return _pythonTypeName; } + get + { + if (Value is null) return false; + + CheckRuntimeIsRunning(); + + using var _ = new Py.GILState(); + return Runtime.PyObject_TypeCheck(Value.Reference, Type.Reference); + } } /// - /// Formats this PythonException object into a message as would be printed - /// out via the Python console. See traceback.format_exception + /// Replaces Value with an instance of Type, if Value is not already an instance of Type. /// - public string Format() + public void Normalize() { - string res; + CheckRuntimeIsRunning(); + IntPtr gs = PythonEngine.AcquireLock(); try { - if (_pyTB != IntPtr.Zero && _pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + if (Exceptions.ErrorOccurred()) throw new InvalidOperationException("Cannot normalize when an error is set"); + + // If an error is set and this PythonException is unnormalized, the error will be cleared and the PythonException will be replaced by a different error. + NewReference value = Value.NewReferenceOrNull(); + NewReference type = Type.NewReferenceOrNull(); + NewReference tb = Traceback.NewReferenceOrNull(); + + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref tb); + + Value = value.MoveToPyObject(); + Type = new PyType(type.Steal()); + try { - IntPtr tb = _pyTB; - IntPtr type = _pyType; - IntPtr value = _pyValue; - - Runtime.XIncref(type); - Runtime.XIncref(value); - Runtime.XIncref(tb); - Runtime.PyErr_NormalizeException(ref type, ref value, ref tb); - - using (PyObject pyType = new PyObject(type)) - using (PyObject pyValue = new PyObject(value)) - using (PyObject pyTB = new PyObject(tb)) - using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + Debug.Assert(Traceback is null == tb.IsNull()); + if (!tb.IsNull()) { - var buffer = new StringBuilder(); - var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); - foreach (PyObject val in values) - { - buffer.Append(val.ToString()); - } - res = buffer.ToString(); + Debug.Assert(Traceback!.Reference == tb); + + int r = Runtime.PyException_SetTraceback(Value.Reference, tb); + ThrowIfIsNotZero(r); } } - else + finally { - res = StackTrace; + tb.Dispose(); } } finally { PythonEngine.ReleaseLock(gs); } - return res; - } - - public bool IsMatches(IntPtr exc) - { - return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; } /// - /// Dispose Method + /// Formats this PythonException object into a message as would be printed + /// out via the Python console. See traceback.format_exception /// - /// - /// The Dispose method provides a way to explicitly release the - /// Python objects represented by a PythonException. - /// If object not properly disposed can cause AppDomain unload issue. - /// See GH#397 and GH#400. - /// - public void Dispose() + public string Format() { - if (!disposed) - { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) - { - IntPtr gs = PythonEngine.AcquireLock(); - if (_pyType != IntPtr.Zero) - { - Runtime.XDecref(_pyType); - _pyType= IntPtr.Zero; - } + CheckRuntimeIsRunning(); - if (_pyValue != IntPtr.Zero) - { - Runtime.XDecref(_pyValue); - _pyValue = IntPtr.Zero; - } + using var _ = new Py.GILState(); - // XXX Do we ever get TraceBack? // - if (_pyTB != IntPtr.Zero) - { - Runtime.XDecref(_pyTB); - _pyTB = IntPtr.Zero; - } - PythonEngine.ReleaseLock(gs); - } - GC.SuppressFinalize(this); - disposed = true; + var copy = Clone(); + copy.Normalize(); + + if (copy.Traceback is null || copy.Value is null) + return StackTrace; + + using var traceback = PyModule.Import("traceback"); + var buffer = new StringBuilder(); + using var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback); + foreach (PyObject val in values) + { + buffer.Append(val); + val.Dispose(); } + return buffer.ToString(); + + } + + public PythonException Clone() + => new PythonException(type: Type, value: Value, traceback: Traceback, + Message, InnerException); + + internal bool Is(IntPtr type) + { + return Runtime.PyErr_GivenExceptionMatches( + given: (Value ?? Type).Reference, + typeOrTypes: new BorrowedReference(type)) != 0; + } + + private static void CheckRuntimeIsRunning() + { + if (!PythonEngine.IsInitialized && Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be running"); } /// - /// Matches Method + /// Returns true if the current Python exception + /// matches the given exception type. /// - /// - /// Returns true if the Python exception type represented by the - /// PythonException instance matches the given exception type. - /// - public static bool Matches(IntPtr ob) + internal static bool CurrentMatches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } - [System.Diagnostics.DebuggerHidden] - public static void ThrowIfIsNull(IntPtr ob) + internal static BorrowedReference ThrowIfIsNull(BorrowedReference ob) { - if (ob == IntPtr.Zero) + if (ob == null) { - throw new PythonException(); + throw ThrowLastAsClrException(); } + + return ob; } - [System.Diagnostics.DebuggerHidden] - internal static void ThrowIfIsNull(BorrowedReference reference) + public static IntPtr ThrowIfIsNull(IntPtr ob) { - if (reference.IsNull) + if (ob == IntPtr.Zero) { - throw new PythonException(); + throw ThrowLastAsClrException(); } + + return ob; } - [System.Diagnostics.DebuggerHidden] public static void ThrowIfIsNotZero(int value) { if (value != 0) { - throw new PythonException(); + throw ThrowLastAsClrException(); } } } diff --git a/src/runtime/pytuple.cs b/src/runtime/pytuple.cs index 530ced3d2..5a18b6bed 100644 --- a/src/runtime/pytuple.cs +++ b/src/runtime/pytuple.cs @@ -77,7 +77,7 @@ private static IntPtr FromArray(PyObject[] items) if (res != 0) { Runtime.Py_DecRef(val); - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } return val; diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs new file mode 100644 index 000000000..e9c80ebf3 --- /dev/null +++ b/src/runtime/pytype.cs @@ -0,0 +1,100 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [Serializable] + public class PyType : PyObject + { + /// Creates heap type object from the . + public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases)) { } + /// Wraps an existing type object. + public PyType(PyObject o) : base(FromObject(o)) { } + + internal PyType(BorrowedReference reference) : base(reference) + { + if (!Runtime.PyType_Check(this.Handle)) + throw new ArgumentException("object is not a type"); + } + + internal PyType(StolenReference reference) : base(EnsureIsType(in reference)) + { + } + + internal new static PyType? FromNullableReference(BorrowedReference reference) + => reference == null + ? null + : new PyType(new NewReference(reference).Steal()); + + internal static PyType FromReference(BorrowedReference reference) + => FromNullableReference(reference) ?? throw new ArgumentNullException(nameof(reference)); + + public string Name + { + get + { + var namePtr = new StrPtr + { + RawPointer = Marshal.ReadIntPtr(Handle, TypeOffset.tp_name), + }; + return namePtr.ToString(System.Text.Encoding.UTF8)!; + } + } + + /// Checks if specified object is a Python type. + public static bool IsType(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return Runtime.PyType_Check(value.obj); + } + + internal IntPtr GetSlot(TypeSlotID slot) + { + IntPtr result = Runtime.PyType_GetSlot(this.Reference, slot); + return Exceptions.ErrorCheckIfNull(result); + } + + private static IntPtr EnsureIsType(in StolenReference reference) + { + IntPtr address = reference.DangerousGetAddressOrNull(); + if (address == IntPtr.Zero) + throw new ArgumentNullException(nameof(reference)); + return EnsureIsType(address); + } + + private static IntPtr EnsureIsType(IntPtr ob) + => Runtime.PyType_Check(ob) + ? ob + : throw new ArgumentException("object is not a type"); + + private static BorrowedReference FromObject(PyObject o) + { + if (o is null) throw new ArgumentNullException(nameof(o)); + if (!IsType(o)) throw new ArgumentException("object is not a type"); + + return o.Reference; + } + + private static IntPtr FromSpec(TypeSpec spec, PyTuple? bases = null) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + if ((spec.Flags & TypeFlags.HeapType) == 0) + throw new NotSupportedException("Only heap types are supported"); + + var nativeSpec = new NativeTypeSpec(spec); + var basesRef = bases is null ? default : bases.Reference; + var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + + PythonException.ThrowIfIsNull(result); + + nativeSpec.Dispose(); + + return result.DangerousMoveToPointer(); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 63467c917..009412ea5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,14 +1,14 @@ -using System.Reflection.Emit; using System; +using System.Diagnostics; using System.Diagnostics.Contracts; using System.Runtime.InteropServices; -using System.Security; using System.Text; using System.Threading; using System.Collections.Generic; using Python.Runtime.Native; using Python.Runtime.Platform; using System.Linq; +using static System.FormattableString; namespace Python.Runtime { @@ -17,51 +17,48 @@ namespace Python.Runtime /// the responsibility of the caller to have acquired the GIL /// before calling any of these methods. /// - public class Runtime + public unsafe class Runtime { - public static int UCS => _UCS; - internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; - -#if PYTHON36 - const string _minor = "6"; -#elif PYTHON37 - const string _minor = "7"; -#elif PYTHON38 - const string _minor = "8"; -#elif PYTHON39 - const string _minor = "9"; -#else -#error You must define one of PYTHON36 to PYTHON39 -#endif + public static string PythonDLL + { + get => _PythonDll; + set + { + if (_isInitialized) + throw new InvalidOperationException("This property must be set before runtime is initialized"); + _PythonDll = value; + } + } -#if WINDOWS - internal const string dllBase = "python3" + _minor; -#else - internal const string dllBase = "python3." + _minor; -#endif + static string _PythonDll = GetDefaultDllName(); + private static string GetDefaultDllName() + { + string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); + if (dll is not null) return dll; -#if PYTHON_WITH_PYDEBUG - internal const string dllWithPyDebug = "d"; -#else - internal const string dllWithPyDebug = ""; -#endif -#if PYTHON_WITH_PYMALLOC - internal const string dllWithPyMalloc = "m"; -#else - internal const string dllWithPyMalloc = ""; -#endif + try + { + LibraryLoader.Instance.GetFunction(IntPtr.Zero, "PyUnicode_GetMax"); + return null; + } catch (MissingMethodException) { } - // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow - // binary substitution of different Python.Runtime.dll builds in a target application. + string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); + if (!Version.TryParse(verString, out var version)) return null; - public static readonly string PythonDLL = _PythonDll; + return GetDefaultDllName(version); + } -#if PYTHON_WITHOUT_ENABLE_SHARED && !NETSTANDARD - internal const string _PythonDll = "__Internal"; -#else - internal const string _PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; -#endif + private static string GetDefaultDllName(Version version) + { + string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Invariant($"{version.Major}{version.Minor}") + : Invariant($"{version.Major}.{version.Minor}"); + string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" + : ".so"; + return prefix + "python" + suffix + ext; + } // set to true when python is finalizing internal static object IsFinalizingLock = new object(); @@ -79,11 +76,6 @@ public class Runtime public static int MainManagedThreadId { get; private set; } - /// - /// Encoding to use to convert Unicode to/from Managed to Native - /// - internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - public static ShutdownMode ShutdownMode { get; internal set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); @@ -93,9 +85,9 @@ internal static Version PyVersion { using (var versionTuple = new PyTuple(PySys_GetObject("version_info"))) { - var major = versionTuple[0].As(); - var minor = versionTuple[1].As(); - var micro = versionTuple[2].As(); + var major = Converter.ToInt32(versionTuple[0].Reference); + var minor = Converter.ToInt32(versionTuple[1].Reference); + var micro = Converter.ToInt32(versionTuple[2].Reference); return new Version(major, minor, micro); } } @@ -140,7 +132,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // If we're coming back from a domain reload or a soft shutdown, // we have previously released the thread state. Restore the main // thread state here. - PyGILState_Ensure(); + if (mode != ShutdownMode.Extension) + { + PyGILState_Ensure(); + } } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; @@ -179,7 +174,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd IntPtr item = PyString_FromString(rtdir); if (PySequence_Contains(path, item) == 0) { - PyList_Append(new BorrowedReference(path), item); + PyList_Append(new BorrowedReference(path), new BorrowedReference(item)); } XDecref(item); AssemblyManager.UpdatePath(); @@ -203,15 +198,15 @@ private static void InitPyMembers() SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), () => PyFalse = IntPtr.Zero); - SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + SetPyMemberTypeOf(ref PyBoolType, PyTrue, () => PyBoolType = IntPtr.Zero); - SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + SetPyMemberTypeOf(ref PyNoneType, PyNone, () => PyNoneType = IntPtr.Zero); - SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + SetPyMemberTypeOf(ref PyTypeType, PyNoneType, () => PyTypeType = IntPtr.Zero); op = PyObject_GetAttrString(builtins, "len"); - SetPyMember(ref PyMethodType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyMethodType, op, () => PyMethodType = IntPtr.Zero); XDecref(op); @@ -220,7 +215,7 @@ private static void InitPyMembers() // // object.__init__ seems safe, though. op = PyObject_GetAttr(PyBaseObjectType, PyIdentifier.__init__); - SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyWrapperDescriptorType, op, () => PyWrapperDescriptorType = IntPtr.Zero); XDecref(op); @@ -231,47 +226,47 @@ private static void InitPyMembers() } op = PyString_FromString("string"); - SetPyMember(ref PyStringType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyStringType, op, () => PyStringType = IntPtr.Zero); XDecref(op); - op = PyUnicode_FromString("unicode"); - SetPyMember(ref PyUnicodeType, PyObject_Type(op), + op = PyString_FromString("unicode"); + SetPyMemberTypeOf(ref PyUnicodeType, op, () => PyUnicodeType = IntPtr.Zero); XDecref(op); - op = PyBytes_FromString("bytes"); - SetPyMember(ref PyBytesType, PyObject_Type(op), + op = EmptyPyBytes(); + SetPyMemberTypeOf(ref PyBytesType, op, () => PyBytesType = IntPtr.Zero); XDecref(op); op = PyTuple_New(0); - SetPyMember(ref PyTupleType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyTupleType, op, () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - SetPyMember(ref PyListType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyListType, op, () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - SetPyMember(ref PyDictType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyDictType, op, () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - SetPyMember(ref PyIntType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyIntType, op, () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - SetPyMember(ref PyLongType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyLongType, op, () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - SetPyMember(ref PyFloatType, PyObject_Type(op), + SetPyMemberTypeOf(ref PyFloatType, op, () => PyFloatType = IntPtr.Zero); XDecref(op); @@ -282,9 +277,9 @@ private static void InitPyMembers() _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); { - IntPtr sys = PyImport_ImportModule("sys"); - PyModuleType = PyObject_Type(sys); - XDecref(sys); + using var sys = PyImport_ImportModule("sys"); + SetPyMemberTypeOf(ref PyModuleType, sys.DangerousGetAddress(), + () => PyModuleType = IntPtr.Zero); } } @@ -363,7 +358,7 @@ internal static void Shutdown(ShutdownMode mode) Finalizer.Shutdown(); InternString.Shutdown(); - if (mode != ShutdownMode.Normal) + if (mode != ShutdownMode.Normal && mode != ShutdownMode.Extension) { PyGC_Collect(); if (mode == ShutdownMode.Soft) @@ -394,7 +389,14 @@ internal static void Shutdown(ShutdownMode mode) else { ResetPyMembers(); - Py_Finalize(); + if (mode != ShutdownMode.Extension) + { + Py_Finalize(); + } + else + { + PyGILState_Release(state); + } } } @@ -419,16 +421,6 @@ internal static ShutdownMode GetDefaultShutdownMode() return ShutdownMode.Normal; } - // called *without* the GIL acquired by clr._AtExit - internal static int AtExit() - { - lock (IsFinalizingLock) - { - IsFinalizing = true; - } - return 0; - } - private static void RunExitFuncs() { PyObject atexit; @@ -436,13 +428,8 @@ private static void RunExitFuncs() { atexit = Py.Import("atexit"); } - catch (PythonException e) + catch (PythonException e) when (e.Is(Exceptions.ImportError)) { - if (!e.IsMatches(Exceptions.ImportError)) - { - throw; - } - e.Dispose(); // The runtime may not provided `atexit` module. return; } @@ -455,7 +442,6 @@ private static void RunExitFuncs() catch (PythonException e) { Console.Error.WriteLine(e); - e.Dispose(); } } } @@ -468,6 +454,12 @@ private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) _pyRefs.Add(value, onRelease); } + private static void SetPyMemberTypeOf(ref IntPtr obj, IntPtr value, Action onRelease) + { + var type = PyObject_Type(new BorrowedReference(value)).DangerousMoveToPointer(); + SetPyMember(ref obj, type, onRelease); + } + private static void ResetPyMembers() { _pyRefs.Release(); @@ -481,9 +473,9 @@ private static void ClearClrModules() for (long i = 0; i < length; i++) { var item = PyList_GetItem(items, i); - var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); - var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); - if (ManagedType.IsManagedType(module)) + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); + if (ManagedType.IsInstanceOfManagedType(module)) { PyDict_DelItem(modules, name); } @@ -498,15 +490,15 @@ private static void RemoveClrRootModule() PyDictTryDelItem(modules, "clr._extra"); } - private static void PyDictTryDelItem(IntPtr dict, string key) + private static void PyDictTryDelItem(BorrowedReference dict, string key) { if (PyDict_DelItemString(dict, key) == 0) { return; } - if (!PythonException.Matches(Exceptions.KeyError)) + if (!PythonException.CurrentMatches(Exceptions.KeyError)) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } PyErr_Clear(); } @@ -528,7 +520,7 @@ private static void MoveClrInstancesOnwershipToPython() obj.CallTypeClear(); // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), // thus just be safe to give it back to GC chain. - if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + if (!_PyObject_GC_IS_TRACKED(obj.ObjectReference)) { PyObject_GC_Track(obj.pyHandle); } @@ -536,6 +528,7 @@ private static void MoveClrInstancesOnwershipToPython() if (obj.gcHandle.IsAllocated) { obj.gcHandle.Free(); + ManagedType.SetGCHandle(obj.ObjectReference, default); } obj.gcHandle = default; } @@ -581,6 +574,8 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; + internal static BorrowedReference CLRMetaType => new BorrowedReference(PyCLRMetaType); + public static PyObject None { get @@ -600,9 +595,9 @@ public static PyObject None /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != null) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } } @@ -733,6 +728,9 @@ internal static IntPtr SelfIncRef(IntPtr op) internal static unsafe void XDecref(IntPtr op) { +#if DEBUG + Debug.Assert(op == IntPtr.Zero || Refcount(op) > 0); +#endif #if !CUSTOM_INCDEC_REF Py_DecRef(op); return; @@ -771,16 +769,12 @@ internal static unsafe void XDecref(IntPtr op) [Pure] internal static unsafe long Refcount(IntPtr op) { -#if PYTHON_WITH_PYDEBUG - var p = (void*)(op + TypeOffset.ob_refcnt); -#else - var p = (void*)op; -#endif - if ((void*)0 == p) + if (op == IntPtr.Zero) { return 0; } - return Is32Bit ? (*(int*)p) : (*(long*)p); + var p = (nint*)(op + ABI.RefCountOffset); + return *p; } /// @@ -788,184 +782,186 @@ internal static unsafe long Refcount(IntPtr op) /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_IncRef(IntPtr ob); + + internal static void Py_IncRef(IntPtr ob) => Delegates.Py_IncRef(ob); /// /// Export of Macro Py_XDecRef. Use XDecref instead. /// Limit this function usage for Testing and Py_Debug builds /// /// PyObject Ptr - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_DecRef(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_Initialize(); + internal static void Py_DecRef(IntPtr ob) => Delegates.Py_DecRef(ob); + + + internal static void Py_Initialize() => Delegates.Py_Initialize(); + + + internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); + internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_IsInitialized(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_Finalize(); + internal static void Py_Finalize() => Delegates.Py_Finalize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_NewInterpreter(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_EndInterpreter(IntPtr threadState); + internal static IntPtr Py_NewInterpreter() => Delegates.Py_NewInterpreter(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_New(IntPtr istate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_Get(); + internal static void Py_EndInterpreter(IntPtr threadState) => Delegates.Py_EndInterpreter(threadState); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyThreadState_UncheckedGet(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThread_get_key_value(IntPtr key); + internal static IntPtr PyThreadState_New(IntPtr istate) => Delegates.PyThreadState_New(istate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThread_get_thread_ident(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyThread_set_key_value(IntPtr key, IntPtr value); + internal static IntPtr PyThreadState_Get() => Delegates.PyThreadState_Get(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyThreadState_Swap(IntPtr key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGILState_Ensure(); + internal static IntPtr _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyGILState_Release(IntPtr gs); + internal static IntPtr PyThread_get_key_value(IntPtr key) => Delegates.PyThread_get_key_value(key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGILState_GetThisThreadState(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - public static extern int Py_Main( - int argc, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv - ); + internal static int PyThread_get_thread_ident() => Delegates.PyThread_get_thread_ident(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_InitThreads(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyEval_ThreadsInitialized(); + internal static int PyThread_set_key_value(IntPtr key, IntPtr value) => Delegates.PyThread_set_key_value(key, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_AcquireLock(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_ReleaseLock(); + internal static IntPtr PyThreadState_Swap(IntPtr key) => Delegates.PyThreadState_Swap(key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_AcquireThread(IntPtr tstate); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_ReleaseThread(IntPtr tstate); + internal static IntPtr PyGILState_Ensure() => Delegates.PyGILState_Ensure(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_SaveThread(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyEval_RestoreThread(IntPtr tstate); + internal static void PyGILState_Release(IntPtr gs) => Delegates.PyGILState_Release(gs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetBuiltins(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetGlobals(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_GetLocals(); + internal static IntPtr PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetProgramName(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetProgramName(IntPtr name); + public static int Py_Main(int argc, string[] argv) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + return Delegates.Py_Main(argc, argvPtr); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } + + internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); + + + internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); + + + internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); + + + internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); + + + internal static void PyEval_AcquireThread(IntPtr tstate) => Delegates.PyEval_AcquireThread(tstate); + + + internal static void PyEval_ReleaseThread(IntPtr tstate) => Delegates.PyEval_ReleaseThread(tstate); + + + internal static IntPtr PyEval_SaveThread() => Delegates.PyEval_SaveThread(); + + + internal static void PyEval_RestoreThread(IntPtr tstate) => Delegates.PyEval_RestoreThread(tstate); + + + internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPythonHome(); + internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetPythonHome(IntPtr home); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPath(); + internal static IntPtr PyEval_GetLocals() => Delegates.PyEval_GetLocals(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_SetPath(IntPtr home); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetVersion(); + internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetPlatform(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetCopyright(); + internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetCompiler(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_GetBuildInfo(); + internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyRun_SimpleString(string code); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); + internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); + + internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); + + + internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); + + + internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); + + + internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); + + + internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); + + + internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); + + + internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); + + const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; + + internal static int PyRun_SimpleString(string code) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); + } + + internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); + } + + internal static IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals) => Delegates.PyEval_EvalCode(co, globals, locals); /// /// Return value: New reference. /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. /// - internal static IntPtr Py_CompileString(string str, string file, int start) + internal static NewReference Py_CompileString(string str, string file, int start) { - return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + using var strPtr = new StrPtr(str, Encoding.UTF8); + using var fileObj = new PyString(file); + return Delegates.Py_CompileStringObject(strPtr, fileObj.Reference, start, Utf8String, -1); } - /// - /// Return value: New reference. - /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. - /// - internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { - return Py_CompileStringExFlags(str, file, start, flags, -1); + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ExecCodeModule(namePtr, code); } - /// - /// Return value: New reference. - /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). - /// - /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); + internal static IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod) => Delegates.PyCFunction_NewEx(ml, self, mod); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCFunction_NewEx(IntPtr ml, IntPtr self, IntPtr mod); + internal static IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw) => Delegates.PyCFunction_Call(func, args, kw); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls); + internal static IntPtr PyMethod_New(IntPtr func, IntPtr self, IntPtr cls) => Delegates.PyMethod_New(func, self, cls); //==================================================================== @@ -985,14 +981,9 @@ internal static unsafe IntPtr PyObject_TYPE(IntPtr op) { return IntPtr.Zero; } -#if PYTHON_WITH_PYDEBUG - var n = 3; -#else - var n = 1; -#endif - return Is32Bit - ? new IntPtr((void*)(*((uint*)p + n))) - : new IntPtr((void*)(*((ulong*)p + n))); + Debug.Assert(TypeOffset.ob_type > 0); + IntPtr* typePtr = (IntPtr*)(op + TypeOffset.ob_type); + return *typePtr; } internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) => new BorrowedReference(PyObject_TYPE(op.DangerousGetAddress())); @@ -1009,61 +1000,104 @@ internal static IntPtr PyObject_Type(IntPtr op) return tp; } + internal static NewReference PyObject_Type(BorrowedReference o) + => Delegates.PyObject_Type(o); + internal static string PyObject_GetTypeName(IntPtr op) { - IntPtr pyType = Marshal.ReadIntPtr(op, ObjectOffset.ob_type); + IntPtr pyType = PyObject_TYPE(op); IntPtr ppName = Marshal.ReadIntPtr(pyType, TypeOffset.tp_name); return Marshal.PtrToStringAnsi(ppName); } + /// + /// Test whether the Python object is an iterable. + /// + internal static bool PyObject_IsIterable(BorrowedReference ob) + => PyObject_IsIterable(ob.DangerousGetAddress()); /// /// Test whether the Python object is an iterable. /// internal static bool PyObject_IsIterable(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iter = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iter); return tp_iter != IntPtr.Zero; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_HasAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name); + internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_HasAttrString(pointer, namePtr); + } + + internal static IntPtr PyObject_GetAttrString(IntPtr pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_GetAttrString(new BorrowedReference(pointer), namePtr) + .DangerousMoveToPointerOrNull(); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_GetAttrString(pointer, namePtr); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, StrPtr name) + => Delegates.PyObject_GetAttrString(pointer, name); + + + internal static int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(pointer, namePtr, value); + } + internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(@object.DangerousGetAddress(), namePtr, value.DangerousGetAddress()); + } + + internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); + + + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) + => Delegates.PyObject_GetAttr(pointer, new BorrowedReference(name)); + internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) + => Delegates.PyObject_GetAttr(new BorrowedReference(pointer), new BorrowedReference(name)) + .DangerousMoveToPointerOrNull(); + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_GetAttr(pointer, name); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetAttrString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string name, IntPtr value); + internal static int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value) => Delegates.PyObject_SetAttr(pointer, name, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_HasAttr(IntPtr pointer, IntPtr name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name); + internal static IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_GetItem(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetAttr(IntPtr pointer, IntPtr name, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetItem(IntPtr pointer, IntPtr key); + internal static int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value) => Delegates.PyObject_SetItem(pointer, key, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_SetItem(IntPtr pointer, IntPtr key, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_DelItem(IntPtr pointer, IntPtr key); + internal static int PyObject_DelItem(IntPtr pointer, IntPtr key) => Delegates.PyObject_DelItem(pointer, key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GetIter(IntPtr op); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw); + internal static IntPtr PyObject_GetIter(IntPtr op) => Delegates.PyObject_GetIter(op); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid); + internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); + internal static IntPtr PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) + => Delegates.PyObject_Call(pointer.DangerousGetAddress(), args.DangerousGetAddress(), kw.DangerousGetAddressOrNull()); + + + internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) + => Delegates.PyObject_CallObject(new BorrowedReference(pointer), new BorrowedReference(args)) + .DangerousMoveToPointerOrNull(); + + + internal static int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); internal static int PyObject_Compare(IntPtr value1, IntPtr value2) { @@ -1090,97 +1124,110 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) return -1; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsInstance(IntPtr ob, IntPtr type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsSubclass(IntPtr ob, IntPtr type); + internal static int PyObject_IsInstance(IntPtr ob, IntPtr type) => Delegates.PyObject_IsInstance(ob, type); + + + internal static int PyObject_IsSubclass(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsSubclass(ob, type); + + + internal static int PyCallable_Check(IntPtr pointer) => Delegates.PyCallable_Check(pointer); + + + internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); + internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); + + + internal static int PyObject_Not(IntPtr pointer) => Delegates.PyObject_Not(pointer); + + internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCallable_Check(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_IsTrue(IntPtr pointer); + internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_Not(IntPtr pointer); - internal static long PyObject_Size(IntPtr pointer) + internal static IntPtr PyObject_Repr(IntPtr pointer) { - return (long)_PyObject_Size(pointer); + AssertNoErorSet(); + + return Delegates.PyObject_Repr(pointer); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Hash(IntPtr op); + internal static IntPtr PyObject_Str(IntPtr pointer) + { + AssertNoErorSet(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Repr(IntPtr pointer); + return Delegates.PyObject_Str(pointer); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Str(IntPtr pointer); + [Conditional("DEBUG")] + internal static void AssertNoErorSet() + { + if (Exceptions.ErrorOccurred()) + throw new InvalidOperationException( + "Can't call with exception set", + PythonException.FetchCurrent()); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyObject_Str")] - internal static extern IntPtr PyObject_Unicode(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Dir(IntPtr pointer); + internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer); -#if PYTHON_WITH_PYDEBUG - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void _Py_NewReference(IntPtr ob); -#endif + internal static void _Py_NewReference(BorrowedReference ob) + { + if (Delegates._Py_NewReference != null) + Delegates._Py_NewReference(ob); + } //==================================================================== // Python buffer API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyBuffer_Release(ref Py_buffer view); + internal static int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, ref view, flags); + + + internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); + + + internal static IntPtr PyBuffer_SizeFromFormat(string format) + { + using var formatPtr = new StrPtr(format, Encoding.ASCII); + return Delegates.PyBuffer_SizeFromFormat(formatPtr); + } + + internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); + + + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - internal static extern IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_IsContiguous(ref Py_buffer view, char order); + internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort); + internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order); + internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags); + + internal static int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); //==================================================================== // Python number API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyNumber_Long")] - internal static extern IntPtr PyNumber_Int(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Long(IntPtr ob); + internal static IntPtr PyNumber_Int(IntPtr ob) => Delegates.PyNumber_Int(ob); + + + internal static IntPtr PyNumber_Long(IntPtr ob) => Delegates.PyNumber_Long(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Float(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyNumber_Check(IntPtr ob); + internal static IntPtr PyNumber_Float(IntPtr ob) => Delegates.PyNumber_Float(ob); + + + internal static bool PyNumber_Check(IntPtr ob) => Delegates.PyNumber_Check(ob); internal static bool PyInt_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, new BorrowedReference(PyIntType)); @@ -1206,33 +1253,26 @@ internal static IntPtr PyInt_FromInt64(long value) return PyInt_FromLong(v); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromLong")] - private static extern IntPtr PyInt_FromLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsLong")] - internal static extern int PyInt_AsLong(IntPtr value); + private static IntPtr PyInt_FromLong(IntPtr value) => Delegates.PyInt_FromLong(value); + + + internal static int PyInt_AsLong(IntPtr value) => Delegates.PyInt_AsLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromString")] - internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromLong(long value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromUnsignedLong")] - internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + internal static IntPtr PyLong_FromLong(long value) => Delegates.PyLong_FromLong(value); + + + internal static IntPtr PyLong_FromUnsignedLong32(uint value) => Delegates.PyLong_FromUnsignedLong32(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_FromUnsignedLong")] - internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong64(ulong value) => Delegates.PyLong_FromUnsignedLong64(value); internal static IntPtr PyLong_FromUnsignedLong(object value) { @@ -1242,44 +1282,42 @@ internal static IntPtr PyLong_FromUnsignedLong(object value) return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromDouble(double value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromLongLong(long value); + internal static IntPtr PyLong_FromDouble(double value) => Delegates.PyLong_FromDouble(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromUnsignedLongLong(ulong value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromString(string value, IntPtr end, int radix); + internal static IntPtr PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsUnsignedLong")] - internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); + internal static IntPtr PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_AsUnsignedLong")] - internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); - internal static object PyLong_AsUnsignedLong(IntPtr value) + internal static IntPtr PyLong_FromString(string value, IntPtr end, int radix) { - if (Is32Bit || IsWindows) - return PyLong_AsUnsignedLong32(value); - else - return PyLong_AsUnsignedLong64(value); + using var valPtr = new StrPtr(value, Encoding.UTF8); + return Delegates.PyLong_FromString(valPtr, end, radix); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern long PyLong_AsLongLong(BorrowedReference value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern long PyLong_AsLongLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern ulong PyLong_AsUnsignedLongLong(IntPtr value); + + internal static nuint PyLong_AsUnsignedSize_t(IntPtr value) => Delegates.PyLong_AsUnsignedSize_t(value); + + internal static nint PyLong_AsSignedSize_t(IntPtr value) => Delegates.PyLong_AsSignedSize_t(new BorrowedReference(value)); + + internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); + + /// + /// This function is a rename of PyLong_AsLongLong, which has a commonly undesired + /// behavior to convert everything (including floats) to integer type, before returning + /// the value as . + /// + /// In most cases you need to check that value is an instance of PyLongObject + /// before using this function using . + /// + + internal static long PyExplicitlyConvertToInt64(IntPtr value) => Delegates.PyExplicitlyConvertToInt64(value); + + internal static ulong PyLong_AsUnsignedLongLong(IntPtr value) => Delegates.PyLong_AsUnsignedLongLong(value); internal static bool PyFloat_Check(IntPtr ob) { @@ -1290,199 +1328,193 @@ internal static bool PyFloat_Check(IntPtr ob) /// Return value: New reference. /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + internal static NewReference PyLong_FromVoidPtr(IntPtr p) => Delegates.PyLong_FromVoidPtr(p); /// /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyFloat_FromDouble(double value); + internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); + + + internal static IntPtr PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); + + + internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); + + + internal static double PyFloat_AsDouble(IntPtr ob) => Delegates.PyFloat_AsDouble(ob); + + + internal static IntPtr PyNumber_Add(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Add(o1, o2); + + + internal static IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Subtract(o1, o2); + + + internal static IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Multiply(o1, o2); + + + internal static IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_TrueDivide(o1, o2); + + + internal static IntPtr PyNumber_And(IntPtr o1, IntPtr o2) => Delegates.PyNumber_And(o1, o2); + + + internal static IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Xor(o1, o2); + + + internal static IntPtr PyNumber_Or(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Or(o1, o2); + + + internal static IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Lshift(o1, o2); + + + internal static IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Rshift(o1, o2); + + + internal static IntPtr PyNumber_Power(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Power(o1, o2); + + + internal static IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_Remainder(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyFloat_FromString(IntPtr value, IntPtr junk); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern double PyFloat_AsDouble(IntPtr ob); + internal static IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Add(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Subtract(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_And(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Xor(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Or(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Lshift(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Rshift(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Power(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceXor(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Remainder(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceOr(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceXor(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceOr(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlacePower(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlacePower(IntPtr o1, IntPtr o2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2); + internal static IntPtr PyNumber_Negative(IntPtr o1) => Delegates.PyNumber_Negative(o1); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Negative(IntPtr o1); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Positive(IntPtr o1); + internal static IntPtr PyNumber_Positive(IntPtr o1) => Delegates.PyNumber_Positive(o1); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Invert(IntPtr o1); + + internal static IntPtr PyNumber_Invert(IntPtr o1) => Delegates.PyNumber_Invert(o1); //==================================================================== // Python sequence API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PySequence_Check(IntPtr pointer); - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } + internal static bool PySequence_Check(IntPtr pointer) => Delegates.PySequence_Check(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); + internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) { return PySequence_SetItem(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + private static int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PySequence_SetItem(pointer, index, value); internal static int PySequence_DelItem(IntPtr pointer, long index) { return PySequence_DelItem(pointer, new IntPtr(index)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); + + private static int PySequence_DelItem(IntPtr pointer, IntPtr index) => Delegates.PySequence_DelItem(pointer, index); internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) { return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + + private static IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) { return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); + + private static int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) { return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - internal static long PySequence_Size(IntPtr pointer) - { - return (long)_PySequence_Size(pointer); - } + private static int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); + + [Obsolete] + internal static nint PySequence_Size(IntPtr pointer) => PySequence_Size(new BorrowedReference(pointer)); + internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); + internal static int PySequence_Contains(IntPtr pointer, IntPtr item) => Delegates.PySequence_Contains(pointer, item); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); + + internal static IntPtr PySequence_Concat(IntPtr pointer, IntPtr other) => Delegates.PySequence_Concat(pointer, other); internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) { return PySequence_Repeat(pointer, new IntPtr(count)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); + private static IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count) => Delegates.PySequence_Repeat(pointer, count); + + + internal static int PySequence_Index(IntPtr pointer, IntPtr item) => Delegates.PySequence_Index(pointer, item); internal static long PySequence_Count(IntPtr pointer, IntPtr value) { return (long)_PySequence_Count(pointer, value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_Tuple(IntPtr pointer); + private static IntPtr _PySequence_Count(IntPtr pointer, IntPtr value) => Delegates._PySequence_Count(pointer, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySequence_List(IntPtr pointer); + + internal static IntPtr PySequence_Tuple(IntPtr pointer) => Delegates.PySequence_Tuple(pointer); + + + internal static IntPtr PySequence_List(IntPtr pointer) => Delegates.PySequence_List(pointer); //==================================================================== // Python string API //==================================================================== - + internal static bool IsStringType(BorrowedReference op) + { + BorrowedReference t = PyObject_TYPE(op); + return (t == new BorrowedReference(PyStringType)) + || (t == new BorrowedReference(PyUnicodeType)); + } internal static bool IsStringType(IntPtr op) { IntPtr t = PyObject_TYPE(op); @@ -1496,92 +1528,76 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { - return PyUnicode_FromKindAndData(_UCS, value, value.Length); + fixed(char* ptr = value) + return Delegates.PyUnicode_DecodeUTF16( + (IntPtr)ptr, + value.Length * sizeof(Char), + IntPtr.Zero, + IntPtr.Zero + ).DangerousMoveToPointerOrNull(); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyBytes_FromString(string op); - internal static long PyBytes_Size(IntPtr op) + internal static IntPtr EmptyPyBytes() { - return (long)_PyBytes_Size(op); + byte* bytes = stackalloc byte[1]; + bytes[0] = 0; + return Delegates.PyBytes_FromString((IntPtr)bytes); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); - - internal static IntPtr PyBytes_AS_STRING(IntPtr ob) + internal static IntPtr PyBytes_AsString(IntPtr ob) => PyBytes_AsString(new BorrowedReference(ob)); + internal static IntPtr PyBytes_AsString(BorrowedReference ob) { - return ob + BytesOffset.ob_sval; + Debug.Assert(ob != null); + return Delegates.PyBytes_AsString(ob); } - - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) + internal static long PyBytes_Size(IntPtr op) { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); + return (long)_PyBytes_Size(op); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); + private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); + + internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode); internal static bool PyUnicode_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyUnicodeType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); + internal static IntPtr PyUnicode_FromObject(IntPtr ob) => Delegates.PyUnicode_FromObject(ob); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( - int kind, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size - ); + internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromKindAndData(_UCS, s, size); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyUnicode_GetMax(); + internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax(); internal static long PyUnicode_GetSize(IntPtr ob) { return (long)_PyUnicode_GetSize(ob); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); + private static IntPtr _PyUnicode_GetSize(IntPtr ob) => Delegates._PyUnicode_GetSize(ob); + + + internal static IntPtr PyUnicode_AsUnicode(IntPtr ob) => Delegates.PyUnicode_AsUnicode(ob); + internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_FromOrdinal(int c); - internal static IntPtr PyUnicode_FromString(string s) + + internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); + + internal static IntPtr PyUnicode_InternFromString(string s) { - return PyUnicode_FromUnicode(s, s.Length); + using var ptr = new StrPtr(s, Encoding.UTF8); + return Delegates.PyUnicode_InternFromString(ptr); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyUnicode_InternFromString(string s); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyUnicode_Compare(IntPtr left, IntPtr right); + internal static int PyUnicode_Compare(IntPtr left, IntPtr right) => Delegates.PyUnicode_Compare(left, right); internal static string GetManagedString(in BorrowedReference borrowedReference) => GetManagedString(borrowedReference.DangerousGetAddress()); @@ -1604,13 +1620,13 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { - IntPtr p = PyUnicode_AsUnicode(op); - int length = (int)PyUnicode_GetSize(op); - - int size = length * _UCS; - var buffer = new byte[size]; - Marshal.Copy(p, buffer, 0, size); - return PyEncoding.GetString(buffer, 0, size); + using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); + var bytesPtr = p.DangerousGetAddress(); + int bytesLength = (int)Runtime.PyBytes_Size(bytesPtr); + char* codePoints = (char*)PyBytes_AsString(bytesPtr); + return new string(codePoints, + startIndex: 1, // skip BOM + length: bytesLength/2-1); // utf16 - BOM } return null; @@ -1626,92 +1642,116 @@ internal static bool PyDict_Check(IntPtr ob) return PyObject_TYPE(ob) == PyDictType; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_New(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + internal static IntPtr PyDict_New() => Delegates.PyDict_New(); + + + internal static int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue) => Delegates.PyDict_Next(p, out ppos, out pkey, out pvalue); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDictProxy_New(IntPtr dict); + + internal static IntPtr PyDictProxy_New(IntPtr dict) => Delegates.PyDictProxy_New(dict); /// /// Return value: Borrowed reference. - /// Return NULL if the key key is not present, but without setting an exception. + /// Return NULL if the key is not present, but without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key); - + internal static IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key) + => Delegates.PyDict_GetItem(new BorrowedReference(pointer), new BorrowedReference(key)) + .DangerousGetAddressOrNull(); /// - /// Return value: Borrowed reference. + /// Return NULL if the key is not present, but without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); + internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); + + internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) + { + using var keyStr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_GetItemString(pointer, keyStr); + } + + internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); /// /// Return 0 on success or -1 on failure. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); + [Obsolete] + internal static int PyDict_SetItem(IntPtr dict, IntPtr key, IntPtr value) => Delegates.PyDict_SetItem(new BorrowedReference(dict), new BorrowedReference(key), new BorrowedReference(value)); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, BorrowedReference key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, key, value); /// /// Return 0 on success or -1 on failure. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_SetItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key, IntPtr value); + internal static int PyDict_SetItemString(IntPtr dict, string key, IntPtr value) + => PyDict_SetItemString(new BorrowedReference(dict), key, new BorrowedReference(value)); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_DelItem(IntPtr pointer, IntPtr key); + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_SetItemString(dict, keyPtr, value); + } + + internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); + + + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_DelItemString(pointer, keyPtr); + } + + internal static int PyMapping_HasKey(IntPtr pointer, IntPtr key) => Delegates.PyMapping_HasKey(pointer, key); + + + [Obsolete] + internal static IntPtr PyDict_Keys(IntPtr pointer) + => Delegates.PyDict_Keys(new BorrowedReference(pointer)) + .DangerousMoveToPointerOrNull(); + internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_DelItemString(IntPtr pointer, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string key); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyMapping_HasKey(IntPtr pointer, IntPtr key); + internal static IntPtr PyDict_Values(IntPtr pointer) => Delegates.PyDict_Values(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Keys(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Values(IntPtr pointer); + internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyDict_Items(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Copy(IntPtr pointer); + internal static IntPtr PyDict_Copy(IntPtr pointer) => Delegates.PyDict_Copy(pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyDict_Update(IntPtr pointer, IntPtr other); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyDict_Clear(IntPtr pointer); + internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); + + + internal static void PyDict_Clear(IntPtr pointer) => Delegates.PyDict_Clear(pointer); internal static long PyDict_Size(IntPtr pointer) { return (long)_PyDict_Size(pointer); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + internal static IntPtr _PyDict_Size(IntPtr pointer) => Delegates._PyDict_Size(pointer); + + + internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); - /// - /// Return value: New reference. - /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PySet_New(IntPtr iterable); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySet_Add(IntPtr set, IntPtr key); + internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); /// /// Return 1 if found, 0 if not found, and -1 if an error is encountered. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); + + internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); //==================================================================== // Python list API @@ -1727,73 +1767,72 @@ internal static IntPtr PyList_New(long size) return PyList_New(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyList_AsTuple(IntPtr pointer); + private static IntPtr PyList_New(IntPtr size) => Delegates.PyList_New(size); + + + internal static IntPtr PyList_AsTuple(IntPtr pointer) => Delegates.PyList_AsTuple(pointer); internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index); + + private static BorrowedReference PyList_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyList_GetItem(pointer, index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) { return PyList_SetItem(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + + private static int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyList_SetItem(pointer, index, value); internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) { return PyList_Insert(pointer, new IntPtr(index), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Append(BorrowedReference pointer, IntPtr value); + private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Reverse(BorrowedReference pointer); + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Sort(BorrowedReference pointer); + + internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); + + + internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) { return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); + + private static IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyList_GetSlice(pointer, start, end); internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) { return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); - internal static long PyList_Size(BorrowedReference pointer) - { - return (long)_PyList_Size(pointer); - } + private static int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value) => Delegates.PyList_SetSlice(pointer, start, end, value); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(BorrowedReference pointer); + internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); //==================================================================== // Python tuple API //==================================================================== + internal static bool PyTuple_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == new BorrowedReference(PyTupleType); + } internal static bool PyTuple_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyTupleType; @@ -1804,138 +1843,175 @@ internal static IntPtr PyTuple_New(long size) return PyTuple_New(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); + + private static IntPtr PyTuple_New(IntPtr size) => Delegates.PyTuple_New(size); internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, long index) => PyTuple_GetItem(pointer, new IntPtr(index)); internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) { - return PyTuple_GetItem(pointer, new IntPtr(index)); + return PyTuple_GetItem(new BorrowedReference(pointer), new IntPtr(index)) + .DangerousGetAddressOrNull(); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); + + private static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, IntPtr index) => Delegates.PyTuple_GetItem(pointer, index); internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) { return PyTuple_SetItem(pointer, new IntPtr(index), value); } + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, StolenReference value) + => PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), value.DangerousGetAddressOrNull()); + + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, BorrowedReference value) + { + var increfValue = value.DangerousGetAddress(); + Runtime.XIncref(increfValue); + return PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), increfValue); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); + private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) { return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - internal static long PyTuple_Size(IntPtr pointer) - { - return (long)_PyTuple_Size(pointer); - } + private static IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end) => Delegates.PyTuple_GetSlice(pointer, start, end); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + internal static nint PyTuple_Size(IntPtr pointer) => PyTuple_Size(new BorrowedReference(pointer)); + internal static nint PyTuple_Size(BorrowedReference pointer) => Delegates.PyTuple_Size(pointer); //==================================================================== // Python iterator API //==================================================================== + internal static bool PyIter_Check(BorrowedReference ob) => PyIter_Check(ob.DangerousGetAddress()); internal static bool PyIter_Check(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iternext = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iternext); return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyIter_Next(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyIter_Next(BorrowedReference pointer); + + internal static IntPtr PyIter_Next(IntPtr pointer) + => Delegates.PyIter_Next(new BorrowedReference(pointer)).DangerousMoveToPointerOrNull(); + internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); //==================================================================== // Python module API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyModule_New(string name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern string PyModule_GetName(IntPtr module); + internal static NewReference PyModule_New(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_New(namePtr); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyModule_GetDict(IntPtr module); + internal static string PyModule_GetName(IntPtr module) + => Delegates.PyModule_GetName(module).ToString(Encoding.UTF8); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern string PyModule_GetFilename(IntPtr module); + internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); -#if PYTHON_WITH_PYDEBUG - [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] -#else - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] -#endif - internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_Import(IntPtr name); + internal static string PyModule_GetFilename(IntPtr module) + => Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8); + + internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver); + + + internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); + + /// + /// We can't use a StolenReference here because the reference is stolen only on success. + /// + /// The module to add the object to. + /// The key that will refer to the object. + /// + /// The object to add to the module. The reference will be stolen only if the + /// method returns 0. + /// + /// Return -1 on error, 0 on success. + internal static int PyModule_AddObject(BorrowedReference module, string name, IntPtr stolenObject) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_AddObject(module, namePtr, stolenObject); + } /// /// Return value: New reference. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ImportModule(string name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_ReloadModule(IntPtr module); + internal static NewReference PyImport_ImportModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ImportModule(namePtr); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_AddModule(string name); + internal static NewReference PyImport_ReloadModule(BorrowedReference module) => Delegates.PyImport_ReloadModule(module); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyImport_GetModuleDict(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PySys_SetArgvEx( - int argc, - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, - int updatepath - ); + internal static BorrowedReference PyImport_AddModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_AddModule(namePtr); + } + + internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); + + + internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + Delegates.PySys_SetArgvEx(argc, argvPtr, updatepath); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } /// /// Return value: Borrowed reference. /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern BorrowedReference PySys_GetObject(string name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PySys_SetObject(string name, IntPtr ob); + internal static BorrowedReference PySys_GetObject(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_GetObject(namePtr); + } + + internal static int PySys_SetObject(string name, BorrowedReference ob) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_SetObject(namePtr, ob); + } //==================================================================== // Python type object API //==================================================================== - internal static bool PyType_Check(IntPtr ob) { return PyObject_TypeCheck(ob, PyTypeType); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyType_Modified(IntPtr type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyType_IsSubtype(IntPtr t1, IntPtr t2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2); + internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); + internal static bool PyType_IsSubtype(BorrowedReference t1, IntPtr ofType) + => PyType_IsSubtype(t1, new BorrowedReference(ofType)); + internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) => Delegates.PyType_IsSubtype(t1, t2); internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) => PyObject_TypeCheck(new BorrowedReference(ob), new BorrowedReference(tp)); @@ -1945,51 +2021,50 @@ internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference return (t == tp) || PyType_IsSubtype(t, tp); } - internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, IntPtr ofType) + => PyType_IsSameAsOrSubtype(type, new BorrowedReference(ofType)); + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedReference ofType) { return (type == ofType) || PyType_IsSubtype(type, ofType); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) - { - return PyType_GenericAlloc(type, new IntPtr(n)); - } + internal static IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw) => Delegates.PyType_GenericNew(type, args, kw); + + internal static IntPtr PyType_GenericAlloc(IntPtr type, nint n) => PyType_GenericAlloc(new BorrowedReference(type), n).DangerousMoveToPointer(); + internal static NewReference PyType_GenericAlloc(BorrowedReference type, nint n) => Delegates.PyType_GenericAlloc(type, n); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + internal static IntPtr PyType_GetSlot(BorrowedReference type, TypeSlotID slot) => Delegates.PyType_GetSlot(type, slot); + internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); /// /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyType_Ready(IntPtr type); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyType_Lookup(IntPtr type, IntPtr name); + internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type); + + + internal static IntPtr _PyType_Lookup(IntPtr type, IntPtr name) => Delegates._PyType_Lookup(type, name); + + + internal static IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name) => Delegates.PyObject_GenericGetAttr(obj, name); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name); + internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr _PyObject_GetDictPtr(IntPtr obj); + internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_Del(IntPtr tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_Track(IntPtr tp); + internal static void PyObject_GC_Track(IntPtr tp) => Delegates.PyObject_GC_Track(tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyObject_GC_UnTrack(IntPtr tp); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void _PyObject_Dump(IntPtr ob); + internal static void PyObject_GC_UnTrack(IntPtr tp) => Delegates.PyObject_GC_UnTrack(tp); + + + internal static void _PyObject_Dump(IntPtr ob) => Delegates._PyObject_Dump(ob); //==================================================================== // Python memory API @@ -2000,70 +2075,87 @@ internal static IntPtr PyMem_Malloc(long size) return PyMem_Malloc(new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); + + private static IntPtr PyMem_Malloc(IntPtr size) => Delegates.PyMem_Malloc(size); internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) { return PyMem_Realloc(ptr, new IntPtr(size)); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyMem_Free(IntPtr ptr); + private static IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size) => Delegates.PyMem_Realloc(ptr, size); + + + internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); //==================================================================== // Python exception API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetString(IntPtr ob, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string message); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject); + internal static void PyErr_SetString(IntPtr ob, string message) + { + using var msgPtr = new StrPtr(message, Encoding.UTF8); + Delegates.PyErr_SetString(ob, msgPtr); + } + + internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); + + + internal static IntPtr PyErr_SetFromErrno(IntPtr ob) => Delegates.PyErr_SetFromErrno(ob); + + + internal static void PyErr_SetNone(IntPtr ob) => Delegates.PyErr_SetNone(ob); + + + internal static int PyErr_ExceptionMatches(IntPtr exception) => Delegates.PyErr_ExceptionMatches(exception); + + + internal static int PyErr_GivenExceptionMatches(BorrowedReference given, BorrowedReference typeOrTypes) => Delegates.PyErr_GivenExceptionMatches(given, typeOrTypes); + + + internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_SetFromErrno(IntPtr ob); + internal static BorrowedReference PyErr_Occurred() => Delegates.PyErr_Occurred(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_SetNone(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyErr_ExceptionMatches(IntPtr exception); + internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyErr_GivenExceptionMatches(IntPtr ob, IntPtr val); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_NormalizeException(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); + internal static void PyErr_Restore(StolenReference type, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(type, val, tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); + internal static void PyErr_Clear() => Delegates.PyErr_Clear(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Clear(); + internal static void PyErr_Print() => Delegates.PyErr_Print(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Print(); + + internal static NewReference PyException_GetCause(BorrowedReference ex) + => Delegates.PyException_GetCause(ex); + internal static NewReference PyException_GetTraceback(BorrowedReference ex) + => Delegates.PyException_GetTraceback(ex); + + /// + /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. + /// + internal static void PyException_SetCause(BorrowedReference ex, StolenReference cause) + => Delegates.PyException_SetCause(ex, cause); + internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) + => Delegates.PyException_SetTraceback(ex, tb); //==================================================================== // Cell API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyCell_Get(BorrowedReference cell); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); + internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); + + + internal static int PyCell_Set(BorrowedReference cell, IntPtr value) => Delegates.PyCell_Set(cell, value); //==================================================================== // Python GC API @@ -2075,14 +2167,14 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyGC_Collect(); - internal static IntPtr _Py_AS_GC(IntPtr ob) + internal static IntPtr PyGC_Collect() => Delegates.PyGC_Collect(); + + internal static IntPtr _Py_AS_GC(BorrowedReference ob) { // XXX: PyGC_Head has a force alignment depend on platform. // See PyGC_Head in objimpl.h for more details. - return Is32Bit ? ob - 16 : ob - 24; + return ob.DangerousGetAddress() - (Is32Bit ? 16 : 24); } internal static IntPtr _Py_FROM_GC(IntPtr gc) @@ -2104,15 +2196,13 @@ internal static IntPtr _PyGCHead_REFS(IntPtr gc) } } - internal static IntPtr _PyGC_REFS(IntPtr ob) + internal static IntPtr _PyGC_REFS(BorrowedReference ob) { return _PyGCHead_REFS(_Py_AS_GC(ob)); } - internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) - { - return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; - } + internal static bool _PyObject_GC_IS_TRACKED(BorrowedReference ob) + => (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; internal static void Py_CLEAR(ref IntPtr ob) { @@ -2124,30 +2214,37 @@ internal static void Py_CLEAR(ref IntPtr ob) // Python Capsules API //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); + internal static NewReference PyCapsule_New(IntPtr pointer, IntPtr name, IntPtr destructor) + => Delegates.PyCapsule_New(pointer, name, destructor); + + internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr name) + { + return Delegates.PyCapsule_GetPointer(capsule, name); + } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); + internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); //==================================================================== // Miscellaneous //==================================================================== - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_Self(IntPtr ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyMethod_Function(IntPtr ob); + internal static IntPtr PyMethod_Self(IntPtr ob) => Delegates.PyMethod_Self(ob); + - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + internal static IntPtr PyMethod_Function(IntPtr ob) => Delegates.PyMethod_Function(ob); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int Py_MakePendingCalls(); + + internal static int Py_AddPendingCall(IntPtr func, IntPtr arg) => Delegates.Py_AddPendingCall(func, arg); + + + internal static int PyThreadState_SetAsyncExcLLP64(uint id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); + + internal static int PyThreadState_SetAsyncExcLP64(ulong id, IntPtr exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); + + + internal static int Py_MakePendingCalls() => Delegates.Py_MakePendingCalls(); internal static void SetNoSiteFlag() { @@ -2182,6 +2279,578 @@ internal static IntPtr GetBuiltins() { return PyImport_Import(PyIdentifier.builtins); } + + internal static class Delegates + { + static readonly ILibraryLoader libraryLoader = LibraryLoader.Instance; + + static Delegates() + { + PyDictProxy_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDictProxy_New), GetUnmanagedDll(_PythonDll)); + Py_IncRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IncRef), GetUnmanagedDll(_PythonDll)); + Py_DecRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_DecRef), GetUnmanagedDll(_PythonDll)); + Py_Initialize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Initialize), GetUnmanagedDll(_PythonDll)); + Py_InitializeEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_InitializeEx), GetUnmanagedDll(_PythonDll)); + Py_IsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IsInitialized), GetUnmanagedDll(_PythonDll)); + Py_Finalize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Finalize), GetUnmanagedDll(_PythonDll)); + Py_NewInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_NewInterpreter), GetUnmanagedDll(_PythonDll)); + Py_EndInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_EndInterpreter), GetUnmanagedDll(_PythonDll)); + PyThreadState_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_New), GetUnmanagedDll(_PythonDll)); + PyThreadState_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Get), GetUnmanagedDll(_PythonDll)); + _PyThreadState_UncheckedGet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyThreadState_UncheckedGet), GetUnmanagedDll(_PythonDll)); + PyThread_get_key_value = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_get_key_value), GetUnmanagedDll(_PythonDll)); + PyThread_get_thread_ident = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_get_thread_ident), GetUnmanagedDll(_PythonDll)); + PyThread_set_key_value = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThread_set_key_value), GetUnmanagedDll(_PythonDll)); + PyThreadState_Swap = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Swap), GetUnmanagedDll(_PythonDll)); + PyGILState_Ensure = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Ensure), GetUnmanagedDll(_PythonDll)); + PyGILState_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Release), GetUnmanagedDll(_PythonDll)); + PyGILState_GetThisThreadState = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_GetThisThreadState), GetUnmanagedDll(_PythonDll)); + Py_Main = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Main), GetUnmanagedDll(_PythonDll)); + PyEval_InitThreads = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_InitThreads), GetUnmanagedDll(_PythonDll)); + PyEval_ThreadsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ThreadsInitialized), GetUnmanagedDll(_PythonDll)); + PyEval_AcquireLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireLock), GetUnmanagedDll(_PythonDll)); + PyEval_ReleaseLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ReleaseLock), GetUnmanagedDll(_PythonDll)); + PyEval_AcquireThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireThread), GetUnmanagedDll(_PythonDll)); + PyEval_ReleaseThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ReleaseThread), GetUnmanagedDll(_PythonDll)); + PyEval_SaveThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_SaveThread), GetUnmanagedDll(_PythonDll)); + PyEval_RestoreThread = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_RestoreThread), GetUnmanagedDll(_PythonDll)); + PyEval_GetBuiltins = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetBuiltins), GetUnmanagedDll(_PythonDll)); + PyEval_GetGlobals = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetGlobals), GetUnmanagedDll(_PythonDll)); + PyEval_GetLocals = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_GetLocals), GetUnmanagedDll(_PythonDll)); + Py_GetProgramName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetProgramName), GetUnmanagedDll(_PythonDll)); + Py_SetProgramName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetProgramName), GetUnmanagedDll(_PythonDll)); + Py_GetPythonHome = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPythonHome), GetUnmanagedDll(_PythonDll)); + Py_SetPythonHome = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetPythonHome), GetUnmanagedDll(_PythonDll)); + Py_GetPath = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPath), GetUnmanagedDll(_PythonDll)); + Py_SetPath = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_SetPath), GetUnmanagedDll(_PythonDll)); + Py_GetVersion = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetVersion), GetUnmanagedDll(_PythonDll)); + Py_GetPlatform = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetPlatform), GetUnmanagedDll(_PythonDll)); + Py_GetCopyright = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCopyright), GetUnmanagedDll(_PythonDll)); + Py_GetCompiler = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetCompiler), GetUnmanagedDll(_PythonDll)); + Py_GetBuildInfo = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_GetBuildInfo), GetUnmanagedDll(_PythonDll)); + PyRun_SimpleStringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_SimpleStringFlags), GetUnmanagedDll(_PythonDll)); + PyRun_StringFlags = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyRun_StringFlags), GetUnmanagedDll(_PythonDll)); + PyEval_EvalCode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_EvalCode), GetUnmanagedDll(_PythonDll)); + Py_CompileStringObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_CompileStringObject), GetUnmanagedDll(_PythonDll)); + PyImport_ExecCodeModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ExecCodeModule), GetUnmanagedDll(_PythonDll)); + PyCFunction_NewEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_NewEx), GetUnmanagedDll(_PythonDll)); + PyCFunction_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCFunction_Call), GetUnmanagedDll(_PythonDll)); + PyMethod_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_New), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_SetAttrString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttrString), GetUnmanagedDll(_PythonDll)); + PyObject_HasAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_HasAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_SetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetItem), GetUnmanagedDll(_PythonDll)); + PyObject_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_SetItem), GetUnmanagedDll(_PythonDll)); + PyObject_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_DelItem), GetUnmanagedDll(_PythonDll)); + PyObject_GetIter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetIter), GetUnmanagedDll(_PythonDll)); + PyObject_Call = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Call), GetUnmanagedDll(_PythonDll)); + PyObject_CallObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_CallObject), GetUnmanagedDll(_PythonDll)); + PyObject_RichCompareBool = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_RichCompareBool), GetUnmanagedDll(_PythonDll)); + PyObject_IsInstance = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsInstance), GetUnmanagedDll(_PythonDll)); + PyObject_IsSubclass = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsSubclass), GetUnmanagedDll(_PythonDll)); + PyCallable_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCallable_Check), GetUnmanagedDll(_PythonDll)); + PyObject_IsTrue = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsTrue), GetUnmanagedDll(_PythonDll)); + PyObject_Not = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Not), GetUnmanagedDll(_PythonDll)); + PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); + PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); + PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); + PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); + PyObject_Type = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Type), GetUnmanagedDll(_PythonDll)); + PyObject_Dir = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll)); + PyObject_GetBuffer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll)); + PyBuffer_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll)); + try + { + PyBuffer_SizeFromFormat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_SizeFromFormat), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + // only in 3.9+ + } + PyBuffer_IsContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_IsContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll)); + PyBuffer_FromContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FromContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_ToContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_ToContiguous), GetUnmanagedDll(_PythonDll)); + PyBuffer_FillContiguousStrides = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FillContiguousStrides), GetUnmanagedDll(_PythonDll)); + PyBuffer_FillInfo = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FillInfo), GetUnmanagedDll(_PythonDll)); + PyNumber_Int = (delegate* unmanaged[Cdecl])GetFunctionByName("PyNumber_Long", GetUnmanagedDll(_PythonDll)); + PyNumber_Long = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Long), GetUnmanagedDll(_PythonDll)); + PyNumber_Float = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Float), GetUnmanagedDll(_PythonDll)); + PyNumber_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Check), GetUnmanagedDll(_PythonDll)); + PyInt_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromLong", GetUnmanagedDll(_PythonDll)); + PyInt_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLong", GetUnmanagedDll(_PythonDll)); + PyLong_FromLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_FromUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromDouble), GetUnmanagedDll(_PythonDll)); + PyLong_FromLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromUnsignedLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromUnsignedLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromString), GetUnmanagedDll(_PythonDll)); + PyLong_AsLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsLong), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLong32 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLong64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsUnsignedLong", GetUnmanagedDll(_PythonDll)); + PyLong_AsLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedLongLong = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsUnsignedLongLong), GetUnmanagedDll(_PythonDll)); + PyLong_FromVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_FromVoidPtr), GetUnmanagedDll(_PythonDll)); + PyLong_AsVoidPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyLong_AsVoidPtr), GetUnmanagedDll(_PythonDll)); + PyFloat_FromDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromDouble), GetUnmanagedDll(_PythonDll)); + PyFloat_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_FromString), GetUnmanagedDll(_PythonDll)); + PyFloat_AsDouble = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyFloat_AsDouble), GetUnmanagedDll(_PythonDll)); + PyNumber_Add = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Add), GetUnmanagedDll(_PythonDll)); + PyNumber_Subtract = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Subtract), GetUnmanagedDll(_PythonDll)); + PyNumber_Multiply = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Multiply), GetUnmanagedDll(_PythonDll)); + PyNumber_TrueDivide = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_TrueDivide), GetUnmanagedDll(_PythonDll)); + PyNumber_And = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_And), GetUnmanagedDll(_PythonDll)); + PyNumber_Xor = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Xor), GetUnmanagedDll(_PythonDll)); + PyNumber_Or = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Or), GetUnmanagedDll(_PythonDll)); + PyNumber_Lshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Lshift), GetUnmanagedDll(_PythonDll)); + PyNumber_Rshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Rshift), GetUnmanagedDll(_PythonDll)); + PyNumber_Power = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Power), GetUnmanagedDll(_PythonDll)); + PyNumber_Remainder = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Remainder), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceAdd = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceAdd), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceSubtract = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceSubtract), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceMultiply = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceMultiply), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceTrueDivide = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceTrueDivide), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceAnd = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceAnd), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceXor = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceXor), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceOr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceOr), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceLshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceLshift), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceRshift = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceRshift), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlacePower = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlacePower), GetUnmanagedDll(_PythonDll)); + PyNumber_InPlaceRemainder = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_InPlaceRemainder), GetUnmanagedDll(_PythonDll)); + PyNumber_Negative = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Negative), GetUnmanagedDll(_PythonDll)); + PyNumber_Positive = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Positive), GetUnmanagedDll(_PythonDll)); + PyNumber_Invert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyNumber_Invert), GetUnmanagedDll(_PythonDll)); + PySequence_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Check), GetUnmanagedDll(_PythonDll)); + PySequence_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_GetItem), GetUnmanagedDll(_PythonDll)); + PySequence_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_SetItem), GetUnmanagedDll(_PythonDll)); + PySequence_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_DelItem), GetUnmanagedDll(_PythonDll)); + PySequence_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_GetSlice), GetUnmanagedDll(_PythonDll)); + PySequence_SetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_SetSlice), GetUnmanagedDll(_PythonDll)); + PySequence_DelSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_DelSlice), GetUnmanagedDll(_PythonDll)); + PySequence_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Size", GetUnmanagedDll(_PythonDll)); + PySequence_Contains = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Contains), GetUnmanagedDll(_PythonDll)); + PySequence_Concat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Concat), GetUnmanagedDll(_PythonDll)); + PySequence_Repeat = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Repeat), GetUnmanagedDll(_PythonDll)); + PySequence_Index = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Index), GetUnmanagedDll(_PythonDll)); + _PySequence_Count = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Count", GetUnmanagedDll(_PythonDll)); + PySequence_Tuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Tuple), GetUnmanagedDll(_PythonDll)); + PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); + PyBytes_AsString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll)); + PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); + _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); + PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); + PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); + _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); + PyUnicode_AsUTF16String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF16String), GetUnmanagedDll(_PythonDll)); + PyUnicode_FromOrdinal = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromOrdinal), GetUnmanagedDll(_PythonDll)); + PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); + PyUnicode_Compare = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_Compare), GetUnmanagedDll(_PythonDll)); + PyDict_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_New), GetUnmanagedDll(_PythonDll)); + PyDict_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Next), GetUnmanagedDll(_PythonDll)); + PyDict_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItem), GetUnmanagedDll(_PythonDll)); + PyDict_GetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItem), GetUnmanagedDll(_PythonDll)); + PyDict_SetItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_SetItemString), GetUnmanagedDll(_PythonDll)); + PyDict_DelItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItem), GetUnmanagedDll(_PythonDll)); + PyDict_DelItemString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_DelItemString), GetUnmanagedDll(_PythonDll)); + PyMapping_HasKey = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMapping_HasKey), GetUnmanagedDll(_PythonDll)); + PyDict_Keys = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Keys), GetUnmanagedDll(_PythonDll)); + PyDict_Values = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Values), GetUnmanagedDll(_PythonDll)); + PyDict_Items = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Items), GetUnmanagedDll(_PythonDll)); + PyDict_Copy = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Copy), GetUnmanagedDll(_PythonDll)); + PyDict_Update = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Update), GetUnmanagedDll(_PythonDll)); + PyDict_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_Clear), GetUnmanagedDll(_PythonDll)); + _PyDict_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyDict_Size", GetUnmanagedDll(_PythonDll)); + PySet_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_New), GetUnmanagedDll(_PythonDll)); + PySet_Add = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_Add), GetUnmanagedDll(_PythonDll)); + PySet_Contains = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySet_Contains), GetUnmanagedDll(_PythonDll)); + PyList_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_New), GetUnmanagedDll(_PythonDll)); + PyList_AsTuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_AsTuple), GetUnmanagedDll(_PythonDll)); + PyList_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetItem), GetUnmanagedDll(_PythonDll)); + PyList_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetItem), GetUnmanagedDll(_PythonDll)); + PyList_Insert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Insert), GetUnmanagedDll(_PythonDll)); + PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); + PyList_Reverse = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Reverse), GetUnmanagedDll(_PythonDll)); + PyList_Sort = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Sort), GetUnmanagedDll(_PythonDll)); + PyList_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetSlice), GetUnmanagedDll(_PythonDll)); + PyList_SetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetSlice), GetUnmanagedDll(_PythonDll)); + PyList_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Size), GetUnmanagedDll(_PythonDll)); + PyTuple_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_New), GetUnmanagedDll(_PythonDll)); + PyTuple_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_GetItem), GetUnmanagedDll(_PythonDll)); + PyTuple_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_SetItem), GetUnmanagedDll(_PythonDll)); + PyTuple_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_GetSlice), GetUnmanagedDll(_PythonDll)); + PyTuple_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyTuple_Size), GetUnmanagedDll(_PythonDll)); + PyIter_Next = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyIter_Next), GetUnmanagedDll(_PythonDll)); + PyModule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_New), GetUnmanagedDll(_PythonDll)); + PyModule_GetName = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll)); + PyModule_GetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll)); + PyModule_GetFilename = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll)); + try + { + PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); + } + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); + PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); + PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); + PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); + PyImport_AddModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_AddModule), GetUnmanagedDll(_PythonDll)); + PyImport_GetModuleDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_GetModuleDict), GetUnmanagedDll(_PythonDll)); + PySys_SetArgvEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetArgvEx), GetUnmanagedDll(_PythonDll)); + PySys_GetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_GetObject), GetUnmanagedDll(_PythonDll)); + PySys_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySys_SetObject), GetUnmanagedDll(_PythonDll)); + PyType_Modified = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Modified), GetUnmanagedDll(_PythonDll)); + PyType_IsSubtype = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_IsSubtype), GetUnmanagedDll(_PythonDll)); + PyType_GenericNew = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericNew), GetUnmanagedDll(_PythonDll)); + PyType_GenericAlloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GenericAlloc), GetUnmanagedDll(_PythonDll)); + PyType_Ready = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Ready), GetUnmanagedDll(_PythonDll)); + _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); + PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL)); + PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); + PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); + PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); + _PyObject_Dump = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_Dump), GetUnmanagedDll(_PythonDll)); + PyMem_Malloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Malloc), GetUnmanagedDll(_PythonDll)); + PyMem_Realloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Realloc), GetUnmanagedDll(_PythonDll)); + PyMem_Free = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Free), GetUnmanagedDll(_PythonDll)); + PyErr_SetString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetString), GetUnmanagedDll(_PythonDll)); + PyErr_SetObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetObject), GetUnmanagedDll(_PythonDll)); + PyErr_SetFromErrno = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetFromErrno), GetUnmanagedDll(_PythonDll)); + PyErr_SetNone = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_SetNone), GetUnmanagedDll(_PythonDll)); + PyErr_ExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_ExceptionMatches), GetUnmanagedDll(_PythonDll)); + PyErr_GivenExceptionMatches = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_GivenExceptionMatches), GetUnmanagedDll(_PythonDll)); + PyErr_NormalizeException = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_NormalizeException), GetUnmanagedDll(_PythonDll)); + PyErr_Occurred = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Occurred), GetUnmanagedDll(_PythonDll)); + PyErr_Fetch = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Fetch), GetUnmanagedDll(_PythonDll)); + PyErr_Restore = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Restore), GetUnmanagedDll(_PythonDll)); + PyErr_Clear = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Clear), GetUnmanagedDll(_PythonDll)); + PyErr_Print = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyErr_Print), GetUnmanagedDll(_PythonDll)); + PyCell_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Get), GetUnmanagedDll(_PythonDll)); + PyCell_Set = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCell_Set), GetUnmanagedDll(_PythonDll)); + PyGC_Collect = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGC_Collect), GetUnmanagedDll(_PythonDll)); + PyCapsule_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_New), GetUnmanagedDll(_PythonDll)); + PyCapsule_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_GetPointer), GetUnmanagedDll(_PythonDll)); + PyCapsule_SetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCapsule_SetPointer), GetUnmanagedDll(_PythonDll)); + PyMethod_Self = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Self), GetUnmanagedDll(_PythonDll)); + PyMethod_Function = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMethod_Function), GetUnmanagedDll(_PythonDll)); + Py_AddPendingCall = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_AddPendingCall), GetUnmanagedDll(_PythonDll)); + Py_MakePendingCalls = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_MakePendingCalls), GetUnmanagedDll(_PythonDll)); + PyLong_AsUnsignedSize_t = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsSize_t", GetUnmanagedDll(_PythonDll)); + PyLong_AsSignedSize_t = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsSsize_t", GetUnmanagedDll(_PythonDll)); + PyExplicitlyConvertToInt64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyLong_AsLongLong", GetUnmanagedDll(_PythonDll)); + PyDict_GetItemWithError = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyDict_GetItemWithError), GetUnmanagedDll(_PythonDll)); + PyException_GetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetCause), GetUnmanagedDll(_PythonDll)); + PyException_GetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_GetTraceback), GetUnmanagedDll(_PythonDll)); + PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); + PyException_SetTraceback = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetTraceback), GetUnmanagedDll(_PythonDll)); + PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyType_GetSlot = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_GetSlot), GetUnmanagedDll(_PythonDll)); + PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); + + try + { + _Py_NewReference = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_Py_NewReference), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) { } + } + + static global::System.IntPtr GetUnmanagedDll(string libraryName) + { + if (libraryName is null) return IntPtr.Zero; + return libraryLoader.Load(libraryName); + } + + static global::System.IntPtr GetFunctionByName(string functionName, global::System.IntPtr libraryHandle) + { + try + { + return libraryLoader.GetFunction(libraryHandle, functionName); + } + catch (MissingMethodException e) when (libraryHandle == IntPtr.Zero) + { + throw new MissingMethodException( + "Did you forget to set Runtime.PythonDLL?" + + " See https://github.com/pythonnet/pythonnet#embedding-python-in-net", + e); + } + } + + internal static delegate* unmanaged[Cdecl] PyDictProxy_New { get; } + internal static delegate* unmanaged[Cdecl] Py_IncRef { get; } + internal static delegate* unmanaged[Cdecl] Py_DecRef { get; } + internal static delegate* unmanaged[Cdecl] Py_Initialize { get; } + internal static delegate* unmanaged[Cdecl] Py_InitializeEx { get; } + internal static delegate* unmanaged[Cdecl] Py_IsInitialized { get; } + internal static delegate* unmanaged[Cdecl] Py_Finalize { get; } + internal static delegate* unmanaged[Cdecl] Py_NewInterpreter { get; } + internal static delegate* unmanaged[Cdecl] Py_EndInterpreter { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_New { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_Get { get; } + internal static delegate* unmanaged[Cdecl] _PyThreadState_UncheckedGet { get; } + internal static delegate* unmanaged[Cdecl] PyThread_get_key_value { get; } + internal static delegate* unmanaged[Cdecl] PyThread_get_thread_ident { get; } + internal static delegate* unmanaged[Cdecl] PyThread_set_key_value { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_Swap { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_Ensure { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_Release { get; } + internal static delegate* unmanaged[Cdecl] PyGILState_GetThisThreadState { get; } + internal static delegate* unmanaged[Cdecl] Py_Main { get; } + internal static delegate* unmanaged[Cdecl] PyEval_InitThreads { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ThreadsInitialized { get; } + internal static delegate* unmanaged[Cdecl] PyEval_AcquireLock { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ReleaseLock { get; } + internal static delegate* unmanaged[Cdecl] PyEval_AcquireThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_ReleaseThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_SaveThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_RestoreThread { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetBuiltins { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetGlobals { get; } + internal static delegate* unmanaged[Cdecl] PyEval_GetLocals { get; } + internal static delegate* unmanaged[Cdecl] Py_GetProgramName { get; } + internal static delegate* unmanaged[Cdecl] Py_SetProgramName { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPythonHome { get; } + internal static delegate* unmanaged[Cdecl] Py_SetPythonHome { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPath { get; } + internal static delegate* unmanaged[Cdecl] Py_SetPath { get; } + internal static delegate* unmanaged[Cdecl] Py_GetVersion { get; } + internal static delegate* unmanaged[Cdecl] Py_GetPlatform { get; } + internal static delegate* unmanaged[Cdecl] Py_GetCopyright { get; } + internal static delegate* unmanaged[Cdecl] Py_GetCompiler { get; } + internal static delegate* unmanaged[Cdecl] Py_GetBuildInfo { get; } + internal static delegate* unmanaged[Cdecl] PyRun_SimpleStringFlags { get; } + internal static delegate* unmanaged[Cdecl] PyRun_StringFlags { get; } + internal static delegate* unmanaged[Cdecl] PyEval_EvalCode { get; } + internal static delegate* unmanaged[Cdecl] Py_CompileStringObject { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ExecCodeModule { get; } + internal static delegate* unmanaged[Cdecl] PyCFunction_NewEx { get; } + internal static delegate* unmanaged[Cdecl] PyCFunction_Call { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_New { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetAttrString { get; } + internal static delegate* unmanaged[Cdecl] PyObject_HasAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetIter { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Call { get; } + internal static delegate* unmanaged[Cdecl] PyObject_CallObject { get; } + internal static delegate* unmanaged[Cdecl] PyObject_RichCompareBool { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsInstance { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsSubclass { get; } + internal static delegate* unmanaged[Cdecl] PyCallable_Check { get; } + internal static delegate* unmanaged[Cdecl] PyObject_IsTrue { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Not { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Size { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Type { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Dir { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GetBuffer { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_SizeFromFormat { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_IsContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FromContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_ToContiguous { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FillContiguousStrides { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_FillInfo { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Int { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Long { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Float { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Check { get; } + internal static delegate* unmanaged[Cdecl] PyInt_FromLong { get; } + internal static delegate* unmanaged[Cdecl] PyInt_AsLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong32 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLong64 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromDouble { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromUnsignedLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong32 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLong64 { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedLongLong { get; } + internal static delegate* unmanaged[Cdecl] PyLong_FromVoidPtr { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsVoidPtr { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_FromDouble { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyFloat_AsDouble { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Add { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Subtract { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Multiply { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_TrueDivide { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_And { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Xor { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Or { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Lshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Rshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Power { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Remainder { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceAdd { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceSubtract { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceMultiply { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceTrueDivide { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceAnd { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceXor { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceOr { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceLshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceRshift { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlacePower { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_InPlaceRemainder { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Negative { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Positive { get; } + internal static delegate* unmanaged[Cdecl] PyNumber_Invert { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Check { get; } + internal static delegate* unmanaged[Cdecl] PySequence_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PySequence_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_SetSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_DelSlice { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Size { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Contains { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Concat { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Repeat { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Index { get; } + internal static delegate* unmanaged[Cdecl] _PySequence_Count { get; } + internal static delegate* unmanaged[Cdecl] PySequence_Tuple { get; } + internal static delegate* unmanaged[Cdecl] PySequence_List { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_AsString { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } + internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } + internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF16String { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_FromOrdinal { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_Compare { get; } + internal static delegate* unmanaged[Cdecl] PyDict_New { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Next { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_SetItemString { get; } + internal static delegate* unmanaged[Cdecl] PyDict_DelItem { get; } + internal static delegate* unmanaged[Cdecl] PyDict_DelItemString { get; } + internal static delegate* unmanaged[Cdecl] PyMapping_HasKey { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Keys { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Values { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Items { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Copy { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Update { get; } + internal static delegate* unmanaged[Cdecl] PyDict_Clear { get; } + internal static delegate* unmanaged[Cdecl] _PyDict_Size { get; } + internal static delegate* unmanaged[Cdecl] PySet_New { get; } + internal static delegate* unmanaged[Cdecl] PySet_Add { get; } + internal static delegate* unmanaged[Cdecl] PySet_Contains { get; } + internal static delegate* unmanaged[Cdecl] PyList_New { get; } + internal static delegate* unmanaged[Cdecl] PyList_AsTuple { get; } + internal static delegate* unmanaged[Cdecl] PyList_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyList_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyList_Insert { get; } + internal static delegate* unmanaged[Cdecl] PyList_Append { get; } + internal static delegate* unmanaged[Cdecl] PyList_Reverse { get; } + internal static delegate* unmanaged[Cdecl] PyList_Sort { get; } + internal static delegate* unmanaged[Cdecl] PyList_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyList_SetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyList_Size { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_New { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_GetItem { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_SetItem { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_GetSlice { get; } + internal static delegate* unmanaged[Cdecl] PyTuple_Size { get; } + internal static delegate* unmanaged[Cdecl] PyIter_Next { get; } + internal static delegate* unmanaged[Cdecl] PyModule_New { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetName { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } + internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } + internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } + internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_AddModule { get; } + internal static delegate* unmanaged[Cdecl] PyImport_GetModuleDict { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetArgvEx { get; } + internal static delegate* unmanaged[Cdecl] PySys_GetObject { get; } + internal static delegate* unmanaged[Cdecl] PySys_SetObject { get; } + internal static delegate* unmanaged[Cdecl] PyType_Modified { get; } + internal static delegate* unmanaged[Cdecl] PyType_IsSubtype { get; } + internal static delegate* unmanaged[Cdecl] PyType_GenericNew { get; } + internal static delegate* unmanaged[Cdecl] PyType_GenericAlloc { get; } + internal static delegate* unmanaged[Cdecl] PyType_Ready { get; } + internal static delegate* unmanaged[Cdecl] _PyType_Lookup { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_Dump { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Malloc { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Realloc { get; } + internal static delegate* unmanaged[Cdecl] PyMem_Free { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetString { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetObject { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetFromErrno { get; } + internal static delegate* unmanaged[Cdecl] PyErr_SetNone { get; } + internal static delegate* unmanaged[Cdecl] PyErr_ExceptionMatches { get; } + internal static delegate* unmanaged[Cdecl] PyErr_GivenExceptionMatches { get; } + internal static delegate* unmanaged[Cdecl] PyErr_NormalizeException { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Occurred { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Fetch { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Restore { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Clear { get; } + internal static delegate* unmanaged[Cdecl] PyErr_Print { get; } + internal static delegate* unmanaged[Cdecl] PyCell_Get { get; } + internal static delegate* unmanaged[Cdecl] PyCell_Set { get; } + internal static delegate* unmanaged[Cdecl] PyGC_Collect { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_New { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyCapsule_SetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_Self { get; } + internal static delegate* unmanaged[Cdecl] PyMethod_Function { get; } + internal static delegate* unmanaged[Cdecl] Py_AddPendingCall { get; } + internal static delegate* unmanaged[Cdecl] Py_MakePendingCalls { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsUnsignedSize_t { get; } + internal static delegate* unmanaged[Cdecl] PyLong_AsSignedSize_t { get; } + internal static delegate* unmanaged[Cdecl] PyExplicitlyConvertToInt64 { get; } + internal static delegate* unmanaged[Cdecl] PyDict_GetItemWithError { get; } + internal static delegate* unmanaged[Cdecl] PyException_GetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_GetTraceback { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } + internal static delegate* unmanaged[Cdecl] PyException_SetTraceback { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } + internal static delegate* unmanaged[Cdecl] PyType_GetSlot { get; } + internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } + internal static delegate* unmanaged[Cdecl] _Py_NewReference { get; } + } } @@ -2191,6 +2860,7 @@ public enum ShutdownMode Normal, Soft, Reload, + Extension, } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index f45e76db4..832e5bbec 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -39,7 +39,7 @@ static void ClearCLRData () BorrowedReference capsule = PySys_GetObject("clr_data"); if (!capsule.IsNull) { - IntPtr oldData = PyCapsule_GetPointer(capsule, null); + IntPtr oldData = PyCapsule_GetPointer(capsule, IntPtr.Zero); PyMem_Free(oldData); PyCapsule_SetPointer(capsule, IntPtr.Zero); } @@ -85,8 +85,9 @@ internal static void Stash() Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); ClearCLRData(); - NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); - PySys_SetObject("clr_data", capsule.DangerousGetAddress()); + + NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); + PySys_SetObject("clr_data", capsule); // Let the dictionary own the reference capsule.Dispose(); } @@ -110,7 +111,7 @@ private static void RestoreRuntimeDataImpl() { return; } - IntPtr mem = PyCapsule_GetPointer(capsule, null); + IntPtr mem = PyCapsule_GetPointer(capsule, IntPtr.Zero); int length = (int)Marshal.ReadIntPtr(mem); byte[] data = new byte[length]; Marshal.Copy(mem + IntPtr.Size, data, 0, length); @@ -118,12 +119,13 @@ private static void RestoreRuntimeDataImpl() var formatter = CreateFormatter(); var storage = (RuntimeDataStorage)formatter.Deserialize(ms); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); RestoreRuntimeDataModules(storage.GetStorage("modules")); TypeManager.RestoreRuntimeData(storage.GetStorage("types")); var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); ImportHook.RestoreRuntimeData(storage.GetStorage("import")); - PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); foreach (var item in objs) { @@ -143,7 +145,7 @@ public static bool HasStashData() public static void ClearStash() { - PySys_SetObject("clr_data", IntPtr.Zero); + PySys_SetObject("clr_data", default); } static bool CheckSerializable (object o) @@ -278,7 +280,7 @@ private static void SaveRuntimeDataModules(RuntimeDataStorage storage) var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); - if (ManagedType.IsManagedType(module)) + if (ManagedType.IsInstanceOfManagedType(module)) { XIncref(name); XIncref(module); @@ -295,7 +297,9 @@ private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) var pyMoudles = PyImport_GetModuleDict(); foreach (var item in modules) { - int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + var moduleName = new BorrowedReference(item.Key); + var module = new BorrowedReference(item.Value); + int res = PyDict_SetItem(pyMoudles, moduleName, module); PythonException.ThrowIfIsNotZero(res); XDecref(item.Key); XDecref(item.Value); diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 69acbcd31..b541a7c44 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -19,20 +19,20 @@ public static void Save() throw new Exception("Runtime State set already"); } - IntPtr objs = IntPtr.Zero; + NewReference objs = default; if (ShouldRestoreObjects) { - objs = PySet_New(IntPtr.Zero); - foreach (var obj in PyGCGetObjects()) + objs = PySet_New(default); + foreach (var objRaw in PyGCGetObjects()) { - AddObjPtrToSet(objs, obj); + AddObjPtrToSet(objs, new BorrowedReference(objRaw)); } } - var modules = PySet_New(IntPtr.Zero); + var modules = PySet_New(default); foreach (var name in GetModuleNames()) { - int res = PySet_Add(modules, name); + int res = PySet_Add(modules, new BorrowedReference(name)); PythonException.ThrowIfIsNotZero(res); } @@ -46,10 +46,9 @@ public static void Save() head->gc.gc_refs = IntPtr.Zero; } { - var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + using var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); int res = PySys_SetObject("dummy_gc", pyDummyGC); PythonException.ThrowIfIsNotZero(res); - XDecref(pyDummyGC); try { @@ -58,7 +57,7 @@ public static void Save() } finally { - XDecref(modules); + modules.Dispose(); } if (ShouldRestoreObjects) @@ -71,7 +70,7 @@ public static void Save() } finally { - XDecref(objs); + objs.Dispose(); } } } @@ -79,8 +78,8 @@ public static void Save() public static void Restore() { - var dummyGCAddr = PySys_GetObject("dummy_gc").DangerousGetAddress(); - if (dummyGCAddr == IntPtr.Zero) + var dummyGCAddr = PySys_GetObject("dummy_gc"); + if (dummyGCAddr.IsNull) { throw new InvalidOperationException("Runtime state have not set"); } @@ -97,9 +96,10 @@ private static void ResotreModules(IntPtr dummyGC) var intialModules = PySys_GetObject("initial_modules"); Debug.Assert(!intialModules.IsNull); var modules = PyImport_GetModuleDict(); - foreach (var name in GetModuleNames()) + foreach (var nameRaw in GetModuleNames()) { - if (PySet_Contains(intialModules.DangerousGetAddress(), name) == 1) + var name = new BorrowedReference(nameRaw); + if (PySet_Contains(intialModules, name) == 1) { continue; } @@ -122,21 +122,15 @@ private static void RestoreObjects(IntPtr dummyGC) { throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); } - IntPtr intialObjs = PySys_GetObject("initial_objs").DangerousGetAddress(); - Debug.Assert(intialObjs != IntPtr.Zero); - foreach (var obj in PyGCGetObjects()) + BorrowedReference intialObjs = PySys_GetObject("initial_objs"); + Debug.Assert(@intialObjs.IsNull); + foreach (var objRaw in PyGCGetObjects()) { - var p = PyLong_FromVoidPtr(obj); - try - { - if (PySet_Contains(intialObjs, p) == 1) - { - continue; - } - } - finally + using var p = PyLong_FromVoidPtr(objRaw); + var obj = new BorrowedReference(objRaw); + if (PySet_Contains(intialObjs, p) == 1) { - XDecref(p); + continue; } Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); ExchangeGCChain(obj, dummyGC); @@ -145,51 +139,42 @@ private static void RestoreObjects(IntPtr dummyGC) public static IEnumerable PyGCGetObjects() { - var gc = PyImport_ImportModule("gc"); - PythonException.ThrowIfIsNull(gc); - var get_objects = PyObject_GetAttrString(gc, "get_objects"); - var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + using var gc = PyModule.Import("gc"); + using var get_objects = gc.GetAttr("get_objects"); + var objs = PyObject_CallObject(get_objects.Handle, IntPtr.Zero); var length = PyList_Size(new BorrowedReference(objs)); for (long i = 0; i < length; i++) { var obj = PyList_GetItem(new BorrowedReference(objs), i); yield return obj.DangerousGetAddress(); } - XDecref(objs); - XDecref(gc); } public static IEnumerable GetModuleNames() { var modules = PyImport_GetModuleDict(); - var names = PyDict_Keys(modules); - var length = PyList_Size(new BorrowedReference(names)); + using var names = PyDict_Keys(modules); + var length = PyList_Size(names); + var result = new IntPtr[length]; for (int i = 0; i < length; i++) { - var name = PyList_GetItem(new BorrowedReference(names), i); - yield return name.DangerousGetAddress(); + result[i] = PyList_GetItem(names, i).DangerousGetAddress(); } - XDecref(names); + return result; } - private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + private static void AddObjPtrToSet(BorrowedReference set, BorrowedReference obj) { - var p = PyLong_FromVoidPtr(obj); - XIncref(obj); - try - { - int res = PySet_Add(set, p); - PythonException.ThrowIfIsNotZero(res); - } - finally - { - XDecref(p); - } + IntPtr objRaw = obj.DangerousGetAddress(); + using var p = PyLong_FromVoidPtr(objRaw); + XIncref(objRaw); + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); } /// /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. /// - private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + private static void ExchangeGCChain(BorrowedReference obj, IntPtr gc) { var head = _Py_AS_GC(obj); if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) diff --git a/src/runtime/tricks/NullOnly.cs b/src/runtime/tricks/NullOnly.cs new file mode 100644 index 000000000..cc2679a61 --- /dev/null +++ b/src/runtime/tricks/NullOnly.cs @@ -0,0 +1,12 @@ +namespace Python.Runtime +{ + /// + /// An utility class, that can only have one value: null. + /// Useful for overloading operators on structs, + /// that have meaningful concept of null value (e.g. pointers and references). + /// + class NullOnly + { + private NullOnly() { } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 973a5aea2..e1bfe6aef 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -21,7 +21,7 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache = new Dictionary(); + private static Dictionary cache = new(); private static readonly Dictionary _slotsHolders = new Dictionary(); private static Dictionary _slotsImpls = new Dictionary(); @@ -45,19 +45,19 @@ internal static void Initialize() internal static void RemoveTypes() { - foreach (var tpHandle in cache.Values) + foreach (var type in cache.Values) { SlotsHolder holder; - if (_slotsHolders.TryGetValue(tpHandle, out holder)) + if (_slotsHolders.TryGetValue(type.Handle, out holder)) { // If refcount > 1, it needs to reset the managed slot, // otherwise it can dealloc without any trick. - if (Runtime.Refcount(tpHandle) > 1) + if (Runtime.Refcount(type.Handle) > 1) { holder.ResetSlots(); } } - Runtime.XDecref(tpHandle); + type.Dispose(); } cache.Clear(); _slotsImpls.Clear(); @@ -68,7 +68,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) { foreach (var tpHandle in cache.Values) { - Runtime.XIncref(tpHandle); + Runtime.XIncref(tpHandle.Handle); } storage.AddValue("cache", cache); storage.AddValue("slots", _slotsImpls); @@ -78,65 +78,57 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - storage.GetValue>("cache", out var _cache); + storage.GetValue>("cache", out var _cache); foreach (var entry in _cache) { if (!entry.Key.Valid) { - Runtime.XDecref(entry.Value); + entry.Value.Dispose(); continue; } Type type = entry.Key.Value;; - IntPtr handle = entry.Value; - cache[type] = handle; - SlotsHolder holder = CreateSolotsHolder(handle); - InitializeSlots(handle, _slotsImpls[type], holder); + cache[type] = entry.Value; + SlotsHolder holder = CreateSolotsHolder(entry.Value.Handle); + InitializeSlots(entry.Value.Handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) } } - /// - /// Return value: Borrowed reference. - /// Given a managed Type derived from ExtensionType, get the handle to - /// a Python type object that delegates its implementation to the Type - /// object. These Python type instances are used to implement internal - /// descriptor and utility types like ModuleObject, PropertyObject, etc. - /// - internal static IntPtr GetTypeHandle(Type type) + internal static PyType GetType(Type type) { // Note that these types are cached with a refcount of 1, so they // effectively exist until the CPython runtime is finalized. - IntPtr handle; - cache.TryGetValue(type, out handle); - if (handle != IntPtr.Zero) + if (!cache.TryGetValue(type, out var pyType)) { - return handle; + pyType = CreateType(type); + cache[type] = pyType; + _slotsImpls.Add(type, type); } - handle = CreateType(type); - cache[type] = handle; - _slotsImpls.Add(type, type); - return handle; + return pyType; } + /// + /// Given a managed Type derived from ExtensionType, get the handle to + /// a Python type object that delegates its implementation to the Type + /// object. These Python type instances are used to implement internal + /// descriptor and utility types like ModuleObject, PropertyObject, etc. + /// + internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; /// - /// Return value: Borrowed reference. - /// Get the handle of a Python type that reflects the given CLR type. + /// Get the Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. /// - internal static IntPtr GetTypeHandle(ManagedType obj, Type type) + internal static PyType GetType(ClassBase obj, Type type) { - IntPtr handle; - cache.TryGetValue(type, out handle); - if (handle != IntPtr.Zero) + if (!cache.TryGetValue(type, out var pyType)) { - return handle; + pyType = CreateType(obj, type); + cache[type] = pyType; + _slotsImpls.Add(type, obj.GetType()); } - handle = CreateType(obj, type); - cache[type] = handle; - _slotsImpls.Add(type, obj.GetType()); - return handle; + return pyType; } @@ -148,44 +140,57 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// - internal static IntPtr CreateType(Type impl) + internal static unsafe PyType CreateType(Type impl) { - IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); - int ob_size = ObjectOffset.Size(type); + IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); + IntPtr base_ = impl == typeof(CLRModule) + ? Runtime.PyModuleType + : Runtime.PyBaseObjectType; + + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + int tp_clr_inst_offset = newFieldOffset; + newFieldOffset += IntPtr.Size; + + int ob_size = newFieldOffset; // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); + Marshal.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); - int flags = TypeFlags.Default | TypeFlags.Managed | + var flags = TypeFlags.Default | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); + // TODO: use PyType(TypeSpec) constructor + var pyType = new PyType(new BorrowedReference(type)); + Runtime.XDecref(type); + + NewReference dict = Runtime.PyObject_GenericGetDict(pyType.Reference); + var mod = NewReference.DangerousFromPointer(Runtime.PyString_FromString("CLR")); Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); - Runtime.XDecref(mod); + mod.Dispose(); + + InitMethods(dict, impl); - InitMethods(type, impl); + dict.Dispose(); // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(type); - return type; + Runtime.PyType_Modified(pyType.Reference); + return pyType; } - internal static IntPtr CreateType(ManagedType impl, Type clrType) + internal static PyType CreateType(ClassBase impl, Type clrType) { // Cleanup the type name to get rid of funny nested type names. string name = $"clr.{clrType.FullName}"; @@ -200,19 +205,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) name = name.Substring(i + 1); } - IntPtr base_ = IntPtr.Zero; - int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - - // XXX Hack, use a different base class for System.Exception - // Python 2.5+ allows new style class exceptions but they *must* - // subclass BaseException (or better Exception). - if (typeof(Exception).IsAssignableFrom(clrType)) - { - ob_size = ObjectOffset.Size(Exceptions.Exception); - } - - int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; - + IntPtr base_ = Runtime.PyBaseObjectType; if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -220,17 +213,32 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) else if (clrType.BaseType != null) { ClassBase bc = ClassManager.GetClass(clrType.BaseType); - base_ = bc.pyHandle; + if (bc.ObjectReference != null) + { + // there are cases when base class has not been fully initialized yet (nested types) + base_ = bc.pyHandle; + } } IntPtr type = AllocateTypeObject(name, Runtime.PyCLRMetaType); - Marshal.WriteIntPtr(type, TypeOffset.ob_type, Runtime.PyCLRMetaType); - Runtime.XIncref(Runtime.PyCLRMetaType); + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + if (ManagedType.IsManagedType(new BorrowedReference(base_))) + { + int baseClrInstOffset = Marshal.ReadInt32(base_, ManagedType.Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); + } + else + { + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); + newFieldOffset += IntPtr.Size; + } + + int ob_size = newFieldOffset; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method SlotsHolder slotsHolder = CreateSolotsHolder(type); @@ -251,24 +259,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // Only set mp_subscript and mp_ass_subscript for types with indexers - if (impl is ClassBase cb) + if (!(impl is ArrayObject)) { - if (!(impl is ArrayObject)) + if (impl.indexer == null || !impl.indexer.CanGet) { - if (cb.indexer == null || !cb.indexer.CanGet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - } - if (cb.indexer == null || !cb.indexer.CanSet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); - } + Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); + } + if (impl.indexer == null || !impl.indexer.CanSet) + { + Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } - } - else - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } if (base_ != IntPtr.Zero) @@ -277,12 +277,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - const int flags = TypeFlags.Default - | TypeFlags.Managed + const TypeFlags flags = TypeFlags.Default + | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.BaseType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note @@ -291,31 +291,58 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + var dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); string mn = clrType.Namespace ?? ""; - IntPtr mod = Runtime.PyString_FromString(mn); + var mod = NewReference.DangerousFromPointer(Runtime.PyString_FromString(mn)); Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); - Runtime.XDecref(mod); + mod.Dispose(); + + var typeRef = new BorrowedReference(type); // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); - Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); + ManagedType.InitGCHandle(typeRef, Runtime.CLRMetaType, gc); // Set the handle attributes on the implementing instance. impl.tpHandle = type; impl.pyHandle = type; //DebugUtil.DumpType(type); + var pyType = new PyType(new BorrowedReference(type)); + Runtime.XDecref(type); + return pyType; + } - return type; + static int InheritOrAllocateStandardFields(IntPtr type, IntPtr @base) + { + int baseSize = Marshal.ReadInt32(@base, TypeOffset.tp_basicsize); + int newFieldOffset = baseSize; + + void InheritOrAllocate(int typeField) + { + int value = Marshal.ReadInt32(@base, typeField); + if (value == 0) + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); + newFieldOffset += IntPtr.Size; + } + else + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); + } + } + + InheritOrAllocate(TypeOffset.tp_dictoffset); + InheritOrAllocate(TypeOffset.tp_weaklistoffset); + + return newFieldOffset; } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, BorrowedReference dictRef) { - var dictRef = new BorrowedReference(py_dict); // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation string name = Runtime.GetManagedString(py_name); @@ -332,7 +359,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { if (Exceptions.ErrorOccurred()) return IntPtr.Zero; } - else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, false)) + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, true)) { return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); } @@ -344,7 +371,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { if (Exceptions.ErrorOccurred()) return IntPtr.Zero; } - else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, false)) + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, true)) { return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); } @@ -362,21 +389,21 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type.Value, - py_dict, + dictRef, (string)namespaceStr, (string)assembly); // create the new ManagedType and python type ClassBase subClass = ClassManager.GetClass(subType); - IntPtr py_type = GetTypeHandle(subClass, subType); + IntPtr py_type = GetType(subClass, subType).Handle; // by default the class dict will have all the C# methods in it, but as this is a // derived class we want the python overrides in there instead if they exist. - IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); - ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, py_dict)); + var cls_dict = new BorrowedReference(Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict)); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, dictRef)); Runtime.XIncref(py_type); // Update the __classcell__ if it exists - var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); + BorrowedReference cell = Runtime.PyDict_GetItemString(cls_dict, "__classcell__"); if (!cell.IsNull) { ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); @@ -445,14 +472,17 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - int size = TypeOffset.magic() + IntPtr.Size; + int size = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + + IntPtr.Size // tp_clr_inst + ; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); - const int flags = TypeFlags.Default - | TypeFlags.Managed + const TypeFlags flags = TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: @@ -463,7 +493,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) if (Runtime.PyType_Ready(type) != 0) { - throw new PythonException(); + throw PythonException.ThrowLastAsClrException(); } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); @@ -472,7 +502,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) // The type has been modified after PyType_Ready has been called // Refresh the type - Runtime.PyType_Modified(type); + Runtime.PyType_Modified(new BorrowedReference(type)); //DebugUtil.DumpType(type); return type; @@ -522,7 +552,7 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => { - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + var tp_dict = new BorrowedReference(Marshal.ReadIntPtr(type, TypeOffset.tp_dict)); if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) { Runtime.PyErr_Print(); @@ -535,60 +565,13 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) - { - // Utility to create a subtype of a std Python type, but with - // a managed type able to override implementation - - IntPtr type = AllocateTypeObject(name, metatype: Runtime.PyTypeType); - //Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); - //Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - - //IntPtr offset = (IntPtr)ObjectOffset.ob_dict; - //Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - - //IntPtr dc = Runtime.PyDict_Copy(dict); - //Marshal.WriteIntPtr(type, TypeOffset.tp_dict, dc); - - Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); - Runtime.XIncref(base_); - - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); - - CopySlot(base_, type, TypeOffset.tp_traverse); - CopySlot(base_, type, TypeOffset.tp_clear); - CopySlot(base_, type, TypeOffset.tp_is_gc); - - SlotsHolder slotsHolder = CreateSolotsHolder(type); - InitializeSlots(type, impl, slotsHolder); - - if (Runtime.PyType_Ready(type) != 0) - { - throw new PythonException(); - } - - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); - - // The type has been modified after PyType_Ready has been called - // Refresh the type - Runtime.PyType_Modified(type); - - return type; - } - - /// /// Utility method to allocate a type object & do basic initialization. /// internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); + PythonException.ThrowIfIsNull(type); // Clr type would not use __slots__, // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), // thus set the ob_size to 0 for avoiding slots iterations. @@ -597,13 +580,15 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. - IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); + + #warning dead code? temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); @@ -615,6 +600,7 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) temp = type + TypeOffset.bf_getbuffer; Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + return type; } @@ -661,50 +647,29 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo { continue; } - var offset = ManagedDataOffsets.GetSlotOffset(slot); + var offset = TypeOffset.GetSlotOffset(slot); Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - /// Can override the slot when it existed - static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder) { - var offset = ManagedDataOffsets.GetSlotOffset(name); - if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + if (!Enum.TryParse(name, out var id)) { - return; + throw new NotSupportedException("Bad slot name " + name); } - Marshal.WriteIntPtr(type, offset, slot); + int offset = TypeOffset.GetSlotOffset(name); + InitializeSlot(type, offset, thunk, slotsHolder); } - static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder) { - int offset = ManagedDataOffsets.GetSlotOffset(name); - - if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) - { - return; - } - Marshal.WriteIntPtr(type, offset, thunk.Address); - if (slotsHolder != null) - { - slotsHolder.Set(offset, thunk); - } + var thunk = Interop.GetThunk(method); + InitializeSlot(type, slotOffset, thunk, slotsHolder); } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + static void InitializeSlot(IntPtr type, int slotOffset, ThunkInfo thunk, SlotsHolder slotsHolder) { - var thunk = Interop.GetThunk(method); Marshal.WriteIntPtr(type, slotOffset, thunk.Address); if (slotsHolder != null) { @@ -712,20 +677,13 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots } } - static bool IsSlotSet(IntPtr type, string name) - { - int offset = ManagedDataOffsets.GetSlotOffset(name); - return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; - } - /// - /// Given a newly allocated Python type object and a managed Type that + /// Given a dict of a newly allocated Python type object and a managed Type that /// implements it, initialize any methods defined by the Type that need /// to appear in the Python type __dict__ (based on custom attribute). /// - private static void InitMethods(IntPtr pytype, Type type) + private static void InitMethods(BorrowedReference typeDict, Type type) { - IntPtr dict = Marshal.ReadIntPtr(pytype, TypeOffset.tp_dict); Type marker = typeof(PythonMethodAttribute); BindingFlags flags = BindingFlags.Public | BindingFlags.Static; @@ -745,7 +703,7 @@ private static void InitMethods(IntPtr pytype, Type type) var mi = new MethodInfo[1]; mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); - Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + Runtime.PyDict_SetItemString(typeDict, method_name, m.ObjectReference); m.DecrRefCount(); addedMethods.Add(method_name); } @@ -785,6 +743,8 @@ class SlotsHolder private List _deallocators = new List(); private bool _alreadyReset = false; + BorrowedReference Type => new BorrowedReference(_type); + /// /// Create slots holder for holding the delegate of slots and be able to reset them. /// @@ -852,15 +812,18 @@ public void ResetSlots() _deallocators.Clear(); // Custom reset - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); - if (handlePtr != IntPtr.Zero) + if (Type != Runtime.CLRMetaType) { - GCHandle handle = GCHandle.FromIntPtr(handlePtr); - if (handle.IsAllocated) + var metatype = Runtime.PyObject_TYPE(Type); + if (ManagedType.TryGetGCHandle(Type, metatype) is { } handle) { - handle.Free(); + if (handle.IsAllocated) + { + handle.Free(); + } + + ManagedType.SetGCHandle(Type, metatype, default); } - Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); } } @@ -913,32 +876,23 @@ static class SlotHelper { public static IntPtr CreateObjectType() { - IntPtr globals = Runtime.PyDict_New(); + using var globals = NewReference.DangerousFromPointer(Runtime.PyDict_New()); if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) { - Runtime.XDecref(globals); - throw new PythonException(); + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); } const string code = "class A(object): pass"; - var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); - IntPtr res = resRef.DangerousGetAddress(); - if (res == IntPtr.Zero) + using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + if (resRef.IsNull()) { - try - { - throw new PythonException(); - } - finally - { - Runtime.XDecref(globals); - } + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); } resRef.Dispose(); - IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); - Debug.Assert(A != IntPtr.Zero); - Runtime.XIncref(A); - Runtime.XDecref(globals); - return A; + BorrowedReference A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(!A.IsNull); + return new NewReference(A).DangerousMoveToPointer(); } } } diff --git a/src/testing/CodecTest.cs b/src/testing/CodecTest.cs new file mode 100644 index 000000000..74f77531b --- /dev/null +++ b/src/testing/CodecTest.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Test +{ + public class ListMember + { + public ListMember(int value, string name) + { + Value = value; + Name = name; + } + + public int Value { get; set; } + public string Name { get; set; } + } + + public class ListConversionTester + { + public int GetLength(IEnumerable o) + { + return o.Count(); + } + public int GetLength(ICollection o) + { + return o.Count; + } + public int GetLength(IList o) + { + return o.Count; + } + public int GetLength2(IEnumerable o) + { + return o.Count(); + } + public int GetLength2(ICollection o) + { + return o.Count; + } + public int GetLength2(IList o) + { + return o.Count; + } + } +} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index e0c0ca8b1..f7bc10bb4 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,13 +1,10 @@ - netstandard2.0 + netstandard2.0;net5.0 + true + true - - - - - diff --git a/src/testing/arraytest.cs b/src/testing/arraytest.cs index 946684962..a3c94e019 100644 --- a/src/testing/arraytest.cs +++ b/src/testing/arraytest.cs @@ -314,4 +314,16 @@ public static Spam[][] EchoRangeAA(Spam[][] items) return items; } } + + public struct Point + { + public Point(float x, float y) + { + X = x; + Y = y; + } + + public float X { get; set; } + public float Y { get; set; } + } } diff --git a/src/testing/delegatetest.cs b/src/testing/delegatetest.cs index e2df9475f..ee66bdad7 100644 --- a/src/testing/delegatetest.cs +++ b/src/testing/delegatetest.cs @@ -13,6 +13,12 @@ namespace Python.Test public delegate bool BoolDelegate(); + public delegate void OutStringDelegate(out string value); + public delegate void RefStringDelegate(ref string value); + public delegate void OutIntDelegate(out int value); + public delegate void RefIntDelegate(ref int value); + public delegate void RefIntRefStringDelegate(ref int intValue, ref string stringValue); + public delegate int IntRefIntRefStringDelegate(ref int intValue, ref string stringValue); public class DelegateTest { @@ -27,6 +33,8 @@ public class DelegateTest public StringDelegate stringDelegate; public ObjectDelegate objectDelegate; public BoolDelegate boolDelegate; + public OutStringDelegate outStringDelegate; + public RefStringDelegate refStringDelegate; public DelegateTest() { @@ -42,6 +50,11 @@ public static string StaticSayHello() return "hello"; } + public void OutHello(out string value) + { + value = "hello"; + } + public string CallStringDelegate(StringDelegate d) { return d(); @@ -56,5 +69,35 @@ public bool CallBoolDelegate(BoolDelegate d) { return d(); } + + public void CallOutIntDelegate(OutIntDelegate d, out int value) + { + d(out value); + } + + public void CallRefIntDelegate(RefIntDelegate d, ref int value) + { + d(ref value); + } + + public void CallOutStringDelegate(OutStringDelegate d, out string value) + { + d(out value); + } + + public void CallRefStringDelegate(RefStringDelegate d, ref string value) + { + d(ref value); + } + + public void CallRefIntRefStringDelegate(RefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + d(ref intValue, ref stringValue); + } + + public int CallIntRefIntRefStringDelegate(IntRefIntRefStringDelegate d, ref int intValue, ref string stringValue) + { + return d(ref intValue, ref stringValue); + } } } diff --git a/src/testing/eventtest.cs b/src/testing/eventtest.cs index dfbd5c881..c9573f71a 100644 --- a/src/testing/eventtest.cs +++ b/src/testing/eventtest.cs @@ -7,7 +7,6 @@ namespace Python.Test /// public delegate void EventHandlerTest(object sender, EventArgsTest e); - #pragma warning disable 67 // Unused events, these are only accessed from Python public class EventTest { @@ -27,6 +26,10 @@ public class EventTest private event EventHandlerTest PrivateEvent; + public event OutStringDelegate OutStringEvent; + public event OutIntDelegate OutIntEvent; + public event RefStringDelegate RefStringEvent; + public event RefIntDelegate RefIntEvent; public static int s_value; public int value; @@ -77,6 +80,27 @@ protected static void OnProtectedStaticEvent(EventArgsTest e) } } + public void OnRefStringEvent(ref string data) + { + RefStringEvent?.Invoke(ref data); + } + + public void OnRefIntEvent(ref int data) + { + RefIntEvent?.Invoke(ref data); + } + + public void OnOutStringEvent(out string data) + { + data = default; + OutStringEvent?.Invoke(out data); + } + + public void OnOutIntEvent(out int data) + { + data = default; + OutIntEvent?.Invoke(out data); + } public void GenericHandler(object sender, EventArgsTest e) { @@ -88,6 +112,26 @@ public static void StaticHandler(object sender, EventArgsTest e) s_value = e.value; } + public void OutStringHandler(out string data) + { + data = value.ToString(); + } + + public void OutIntHandler(out int data) + { + data = value; + } + + public void RefStringHandler(ref string data) + { + data += "!"; + } + + public void RefIntHandler(ref int data) + { + data++; + } + public static void ShutUpCompiler() { // Quiet compiler warnings. diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index 1e9c71ac6..238435811 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -35,6 +35,9 @@ public DerivedFromOpenGeneric(int arg1, V arg2, W arg3) : base(arg1, arg2) } } + public class GenericTypeWithConstraint + where T: struct + { } public class GenericNameTest1 { @@ -125,4 +128,12 @@ public static string Overloaded(int arg1, int arg2, string arg3) return arg3; } } + + public class GenericArrayConversionTest + { + public static T[] EchoRange(T[] items) + { + return items; + } + } } diff --git a/src/testing/indexertest.cs b/src/testing/indexertest.cs index 78bb99a7c..08e6ad053 100644 --- a/src/testing/indexertest.cs +++ b/src/testing/indexertest.cs @@ -427,7 +427,8 @@ public interface IIndexer public interface IInheritedIndexer : IIndexer { } - public class InterfaceInheritedIndexerTest :IndexerBase, IInheritedIndexer { + public class InterfaceInheritedIndexerTest : IndexerBase, IInheritedIndexer + { private System.Collections.Generic.IDictionary d = new System.Collections.Generic.Dictionary(); public string this[int index] @@ -436,4 +437,13 @@ public string this[int index] set { t[index] = value; } } } + + public class PublicInheritedOverloadedIndexer : PublicIndexerTest + { + public string this[string index] + { + get { return GetValue(index); } + set { t[index] = value; } + } + } } diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index f5d694488..abdc5f4d6 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -85,7 +85,7 @@ public Type[] TestNullArrayConversion(Type[] v) public static string[] TestStringParamsArg(params string[] args) { - return args.Concat(new []{"tail"}).ToArray(); + return args.Concat(new[] { "tail" }).ToArray(); } public static object[] TestObjectParamsArg(params object[] args) @@ -654,32 +654,32 @@ public static string Casesensitive() return "Casesensitive"; } - public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + public static string DefaultParams(int a = 0, int b = 0, int c = 0, int d = 0) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + public static string OptionalParams([Optional] int a, [Optional] int b, [Optional] int c, [Optional] int d) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static bool OptionalParams_TestMissing([Optional]object a) + public static bool OptionalParams_TestMissing([Optional] object a) { return a == Type.Missing; } - public static bool OptionalParams_TestReferenceType([Optional]string a) + public static bool OptionalParams_TestReferenceType([Optional] string a) { return a == null; } - public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + public static string OptionalAndDefaultParams([Optional] int a, [Optional] int b, int c = 0, int d = 0) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } - public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + public static string OptionalAndDefaultParams2([Optional] int a, [Optional] int b, [Optional, DefaultParameterValue(1)] int c, int d = 2) { return string.Format("{0}{1}{2}{3}", a, b, c, d); } @@ -717,6 +717,13 @@ public static string ParamsArrayOverloaded(int i, params int[] paramsArray) public static void EncodingTestÅngström() { } + + // This method can never be invoked from Python, but we want to test that any attempt fails gracefully instead of crashing. + unsafe + public static void PointerArray(int*[] array) + { + + } } diff --git a/src/testing/threadtest.cs b/src/testing/threadtest.cs index 9c76929b2..6664c3643 100644 --- a/src/testing/threadtest.cs +++ b/src/testing/threadtest.cs @@ -34,7 +34,7 @@ public static string CallEchoString(string arg) { if (module == null) { - module = PythonEngine.ModuleFromString("tt", testmod); + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring"); var parg = new PyString(arg); @@ -58,7 +58,7 @@ public static string CallEchoString2(string arg) { if (module == null) { - module = PythonEngine.ModuleFromString("tt", testmod); + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring2"); diff --git a/src/tests/conftest.py b/src/tests/conftest.py deleted file mode 100644 index 17085e3ef..000000000 --- a/src/tests/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -# TODO: move tests one out of src to project root. -# TODO: travis has numpy on their workers. Maybe add tests? - -"""Helpers for testing.""" - -import ctypes -import os -import sys -import sysconfig - -import pytest - -# Add path for `Python.Test` -cwd = os.path.dirname(__file__) -fixtures_path = os.path.join(cwd, "fixtures") - -BUILD_TEST = True -if BUILD_TEST: - from subprocess import check_call - test_proj_path = os.path.join(cwd, "..", "testing") - check_call(["dotnet", "build", test_proj_path, "-o", fixtures_path]) - -sys.path.append(fixtures_path) - -import clr - -# Add References for tests -clr.AddReference("Python.Test") -clr.AddReference("System.Collections") -clr.AddReference("System.Data") - - -def pytest_report_header(config): - """Generate extra report headers""" - # FIXME: https://github.com/pytest-dev/pytest/issues/2257 - is_64bits = sys.maxsize > 2**32 - arch = "x64" if is_64bits else "x86" - ucs = ctypes.sizeof(ctypes.c_wchar) - libdir = sysconfig.get_config_var("LIBDIR") - shared = bool(sysconfig.get_config_var("Py_ENABLE_SHARED")) - - header = ("Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}, " - "Py_ENABLE_SHARED: {shared}".format(**locals())) - return header - - -@pytest.fixture() -def filepath(): - """Returns full filepath for file in `fixtures` directory.""" - - def make_filepath(filename): - # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture - return os.path.join(fixtures_path, filename) - - return make_filepath diff --git a/src/tests/fixtures/.gitkeep b/src/tests/fixtures/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/tests/fixtures/netstandard2.0/.gitkeep b/src/tests/fixtures/netstandard2.0/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/tests/test_codec.py b/src/tests/test_codec.py new file mode 100644 index 000000000..9fdbfbbf5 --- /dev/null +++ b/src/tests/test_codec.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +"""Test conversions using codecs from client python code""" +import clr +import System +import pytest +import Python.Runtime +from Python.Test import ListConversionTester, ListMember + +class int_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return self.counter + +class obj_iterable(): + def __init__(self): + self.counter = 0 + def __iter__(self): + return self + def __next__(self): + if self.counter == 3: + raise StopIteration + self.counter = self.counter + 1 + return ListMember(self.counter, "Number " + str(self.counter)) + +def test_iterable(): + """Test that a python iterable can be passed into a function that takes an IEnumerable""" + + #Python.Runtime.Codecs.ListDecoder.Register() + #Python.Runtime.Codecs.SequenceDecoder.Register() + Python.Runtime.Codecs.IterableDecoder.Register() + ob = ListConversionTester() + + iterable = int_iterable() + assert 3 == ob.GetLength(iterable) + + iterable2 = obj_iterable() + assert 3 == ob.GetLength2(iterable2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_sequence(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + tup = (1,2,3) + assert 3 == ob.GetLength(tup) + + tup2 = (ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")) + assert 3 == ob.GetLength(tup2) + + Python.Runtime.PyObjectConversions.Reset() + +def test_list(): + Python.Runtime.Codecs.SequenceDecoder.Register() + ob = ListConversionTester() + + l = [1,2,3] + assert 3 == ob.GetLength(l) + + l2 = [ListMember(1, "one"), ListMember(2, "two"), ListMember(3, "three")] + assert 3 == ob.GetLength(l2) + + Python.Runtime.PyObjectConversions.Reset() diff --git a/src/tests/__init__.py b/tests/__init__.py similarity index 100% rename from src/tests/__init__.py rename to tests/__init__.py diff --git a/src/tests/_missing_import.py b/tests/_missing_import.py similarity index 100% rename from src/tests/_missing_import.py rename to tests/_missing_import.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..0361830d6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# TODO: move tests one out of src to project root. +# TODO: travis has numpy on their workers. Maybe add tests? + +"""Helpers for testing.""" + +import ctypes +import os +import sys +import sysconfig +from subprocess import check_call +from tempfile import mkdtemp +import shutil + +import pytest + +from pythonnet import set_runtime + +# Add path for `Python.Test` +cwd = os.path.dirname(__file__) +fixtures_path = os.path.join(cwd, "fixtures") +sys.path.append(fixtures_path) + +def pytest_addoption(parser): + parser.addoption( + "--runtime", + action="store", + default="default", + help="Must be one of default, netcore, netfx and mono" + ) + +collect_ignore = [] + +def pytest_configure(config): + global bin_path + if "clr" in sys.modules: + # Already loaded (e.g. by the C# test runner), skip build + import clr + clr.AddReference("Python.Test") + return + + runtime_opt = config.getoption("runtime") + + test_proj_path = os.path.join(cwd, "..", "src", "testing") + + if runtime_opt not in ["netcore", "netfx", "mono", "default"]: + raise RuntimeError(f"Invalid runtime: {runtime_opt}") + + bin_path = mkdtemp() + + # tmpdir_factory.mktemp(f"pythonnet-{runtime_opt}") + + fw = "net5.0" if runtime_opt == "netcore" else "netstandard2.0" + + check_call(["dotnet", "publish", "-f", fw, "-o", bin_path, test_proj_path]) + + sys.path.append(bin_path) + + if runtime_opt == "default": + pass + elif runtime_opt == "netfx": + from clr_loader import get_netfx + runtime = get_netfx() + set_runtime(runtime) + elif runtime_opt == "mono": + from clr_loader import get_mono + runtime = get_mono() + set_runtime(runtime) + elif runtime_opt == "netcore": + from clr_loader import get_coreclr + rt_config_path = os.path.join(bin_path, "Python.Test.runtimeconfig.json") + runtime = get_coreclr(rt_config_path) + set_runtime(runtime) + + import clr + clr.AddReference("Python.Test") + + soft_mode = False + try: + os.environ['PYTHONNET_SHUTDOWN_MODE'] == 'Soft' + except: pass + + if config.getoption("--runtime") == "netcore" or soft_mode\ + : + collect_ignore.append("domain_tests/test_domain_reload.py") + else: + domain_tests_dir = os.path.join(os.path.dirname(__file__), "domain_tests") + bin_path = os.path.join(domain_tests_dir, "bin") + check_call(["dotnet", "build", domain_tests_dir, "-o", bin_path]) + + + + +def pytest_unconfigure(config): + global bin_path + try: + shutil.rmtree(bin_path) + except Exception: + pass + +def pytest_report_header(config): + """Generate extra report headers""" + # FIXME: https://github.com/pytest-dev/pytest/issues/2257 + is_64bits = sys.maxsize > 2**32 + arch = "x64" if is_64bits else "x86" + ucs = ctypes.sizeof(ctypes.c_wchar) + libdir = sysconfig.get_config_var("LIBDIR") + shared = bool(sysconfig.get_config_var("Py_ENABLE_SHARED")) + + header = ("Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}, " + "Py_ENABLE_SHARED: {shared}".format(**locals())) + return header + + +@pytest.fixture() +def filepath(): + """Returns full filepath for file in `fixtures` directory.""" + + def make_filepath(filename): + # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture + return os.path.join(fixtures_path, filename) + + return make_filepath diff --git a/src/domain_tests/App.config b/tests/domain_tests/App.config similarity index 100% rename from src/domain_tests/App.config rename to tests/domain_tests/App.config diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/tests/domain_tests/Python.DomainReloadTests.csproj similarity index 89% rename from src/domain_tests/Python.DomainReloadTests.csproj rename to tests/domain_tests/Python.DomainReloadTests.csproj index 54196f210..9cb61c6f4 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/tests/domain_tests/Python.DomainReloadTests.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs similarity index 97% rename from src/domain_tests/TestRunner.cs rename to tests/domain_tests/TestRunner.cs index a21297829..66fb4f894 100644 --- a/src/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -30,6 +30,11 @@ namespace Python.DomainReloadTests /// which test case to run. That's because pytest assumes we'll run /// everything in one process, but we really want a clean process on each /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// ### Debugging tips: ### + /// * Running pytest with the `-s` argument prevents stdout capture by pytest + /// * Add a sleep into the python test case before the crash/failure, then while + /// sleeping, attach the debugger to the Python.TestDomainReload.exe process. /// /// class TestRunner @@ -1092,6 +1097,29 @@ assert sys.my_obj is not None foo = sys.my_obj.Inner() print(foo) + ", + }, + new TestCase + { + // The C# code for this test doesn't matter; we're testing + // that the import hook behaves properly after a domain reload + Name = "import_after_reload", + DotNetBefore = "", + DotNetAfter = "", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + + +def after_reload(): + assert 'System' in sys.modules + assert 'clr' in sys.modules + import clr + import System + ", }, }; @@ -1264,7 +1292,13 @@ static string CreateAssembly(string name, string code, bool exe = false) } parameters.ReferencedAssemblies.Add(netstandard); parameters.ReferencedAssemblies.Add(PythonDllLocation); - CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + // Write code to file so it can debugged. + var sourcePath = Path.Combine(TestPath, name+"_source.cs"); + using(var file = new StreamWriter(sourcePath)) + { + file.Write(code); + } + CompilerResults results = provider.CompileAssemblyFromFile(parameters, sourcePath); if (results.NativeCompilerReturnValue != 0) { var stderr = System.Console.Error; diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py new file mode 100644 index 000000000..e7a82ded2 --- /dev/null +++ b/tests/domain_tests/test_domain_reload.py @@ -0,0 +1,93 @@ +import subprocess +import os +import platform + +import pytest + +from pythonnet.find_libpython import find_libpython +libpython = find_libpython() + +pytestmark = pytest.mark.xfail(libpython is None, reason="Can't find suitable libpython") + + +def _run_test(testname): + dirname = os.path.split(__file__)[0] + exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') + args = [exename, testname] + + if platform.system() != 'Windows': + args = ['mono'] + args + + env = os.environ.copy() + env["PYTHONNET_PYDLL"] = libpython + + proc = subprocess.Popen(args, env=env) + proc.wait() + + assert proc.returncode == 0 + +def test_rename_class(): + _run_test('class_rename') + +def test_rename_class_member_static_function(): + _run_test('static_member_rename') + +def test_rename_class_member_function(): + _run_test('member_rename') + +def test_rename_class_member_field(): + _run_test('field_rename') + +def test_rename_class_member_property(): + _run_test('property_rename') + +def test_rename_namespace(): + _run_test('namespace_rename') + +def test_field_visibility_change(): + _run_test("field_visibility_change") + +def test_method_visibility_change(): + _run_test("method_visibility_change") + +def test_property_visibility_change(): + _run_test("property_visibility_change") + +def test_class_visibility_change(): + _run_test("class_visibility_change") + +@pytest.mark.skip(reason='FIXME: Domain reload fails when Python points to a .NET object which points back to Python objects') +def test_method_parameters_change(): + _run_test("method_parameters_change") + +def test_method_return_type_change(): + _run_test("method_return_type_change") + +def test_field_type_change(): + _run_test("field_type_change") + +@pytest.mark.xfail(reason="Events not yet serializable") +def test_rename_event(): + _run_test('event_rename') + +@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") +def test_construct_removed_class(): + _run_test("construct_removed_class") + +def test_out_to_ref_param(): + _run_test("out_to_ref_param") + +def test_ref_to_out_param(): + _run_test("ref_to_out_param") + +def test_ref_to_in_param(): + _run_test("ref_to_in_param") + +def test_in_to_ref_param(): + _run_test("in_to_ref_param") + +def test_nested_type(): + _run_test("nested_type") + +def test_import_after_reload(): + _run_test("import_after_reload") \ No newline at end of file diff --git a/src/tests/fixtures/argv-fixture.py b/tests/fixtures/argv-fixture.py similarity index 100% rename from src/tests/fixtures/argv-fixture.py rename to tests/fixtures/argv-fixture.py diff --git a/src/tests/importtest.py b/tests/importtest.py similarity index 100% rename from src/tests/importtest.py rename to tests/importtest.py diff --git a/src/tests/leaktest.py b/tests/leaktest.py similarity index 100% rename from src/tests/leaktest.py rename to tests/leaktest.py diff --git a/src/tests/profile.py b/tests/profile.py similarity index 100% rename from src/tests/profile.py rename to tests/profile.py diff --git a/src/tests/runtests.py b/tests/runtests.py similarity index 100% rename from src/tests/runtests.py rename to tests/runtests.py diff --git a/src/tests/stress.py b/tests/stress.py similarity index 100% rename from src/tests/stress.py rename to tests/stress.py diff --git a/src/tests/stresstest.py b/tests/stresstest.py similarity index 100% rename from src/tests/stresstest.py rename to tests/stresstest.py diff --git a/src/tests/test_array.py b/tests/test_array.py similarity index 97% rename from src/tests/test_array.py rename to tests/test_array.py index 9ab044b29..d6f08a961 100644 --- a/src/tests/test_array.py +++ b/tests/test_array.py @@ -680,7 +680,7 @@ def test_enum_array(): items[-1] = ShortEnum.Zero assert items[-1] == ShortEnum.Zero - with pytest.raises(ValueError): + with pytest.raises(TypeError): ob = Test.EnumArrayTest() ob.items[0] = 99 @@ -836,8 +836,15 @@ def test_typed_array(): with pytest.raises(TypeError): ob = Test.TypedArrayTest() - ob.items["wrong"] = "wrong" + ob.items["wrong"] = Spam("0") + with pytest.raises(TypeError): + ob = Test.TypedArrayTest() + _ = ob.items[0.5] + + with pytest.raises(TypeError): + ob = Test.TypedArrayTest() + ob.items[0.5] = Spam("0") def test_multi_dimensional_array(): """Test multi-dimensional arrays.""" @@ -901,8 +908,23 @@ def test_multi_dimensional_array(): with pytest.raises(TypeError): ob = Test.MultiDimensionalArrayTest() - ob[0, 0] = "wrong" + ob.items[0, 0] = "wrong" + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob["0", 0] = 0 + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob.items["0", 0] = 0 + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + _ = ob.items[0.5, 0] + + with pytest.raises(TypeError): + ob = Test.MultiDimensionalArrayTest() + ob.items[0.5, 0] = 0 def test_array_iteration(): """Test array iteration.""" @@ -1144,10 +1166,8 @@ def test_boxed_value_type_mutation_result(): # to accidentally write code like the following which is not really # mutating value types in-place but changing boxed copies. - clr.AddReference('System.Drawing') - - from System.Drawing import Point from System import Array + from Python.Test import Point items = Array.CreateInstance(Point, 5) @@ -1189,6 +1209,15 @@ def test_create_array_from_shape(): with pytest.raises(ValueError): Array[int](-1) + with pytest.raises(TypeError): + Array[int]('1') + + with pytest.raises(ValueError): + Array[int](-1, -1) + + with pytest.raises(TypeError): + Array[int]('1', '1') + def test_special_array_creation(): """Test using the Array[] syntax for creating arrays.""" from Python.Test import ISayHello1, InterfaceTest, ShortEnum diff --git a/src/tests/test_callback.py b/tests/test_callback.py similarity index 100% rename from src/tests/test_callback.py rename to tests/test_callback.py diff --git a/src/tests/test_class.py b/tests/test_class.py similarity index 99% rename from src/tests/test_class.py rename to tests/test_class.py index 4666631f7..f961b3975 100644 --- a/src/tests/test_class.py +++ b/tests/test_class.py @@ -126,9 +126,7 @@ def __init__(self, v): def test_struct_construction(): """Test construction of structs.""" - clr.AddReference('System.Drawing') - - from System.Drawing import Point + from Python.Test import Point p = Point() assert p.X == 0 diff --git a/src/tests/test_clrmethod.py b/tests/test_clrmethod.py similarity index 100% rename from src/tests/test_clrmethod.py rename to tests/test_clrmethod.py diff --git a/src/tests/test_constructors.py b/tests/test_constructors.py similarity index 100% rename from src/tests/test_constructors.py rename to tests/test_constructors.py diff --git a/src/tests/test_conversion.py b/tests/test_conversion.py similarity index 94% rename from src/tests/test_conversion.py rename to tests/test_conversion.py index 313274647..eec2bcde6 100644 --- a/src/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -343,7 +343,7 @@ def test_uint32_conversion(): ob.UInt32Field = System.UInt32(0) assert ob.UInt32Field == 0 - with pytest.raises(ValueError): + with pytest.raises(TypeError): ConversionTest().UInt32Field = "spam" with pytest.raises(TypeError): @@ -382,7 +382,10 @@ def test_uint64_conversion(): ob.UInt64Field = System.UInt64(0) assert ob.UInt64Field == 0 - with pytest.raises(ValueError): + with pytest.raises(TypeError): + ConversionTest().UInt64Field = 0.5 + + with pytest.raises(TypeError): ConversionTest().UInt64Field = "spam" with pytest.raises(TypeError): @@ -403,8 +406,8 @@ def test_uint64_conversion(): def test_single_conversion(): """Test single conversion.""" - assert System.Single.MaxValue == 3.402823e38 - assert System.Single.MinValue == -3.402823e38 + assert System.Single.MaxValue == pytest.approx(3.402823e38) + assert System.Single.MinValue == pytest.approx(-3.402823e38) ob = ConversionTest() assert ob.SingleField == 0.0 @@ -598,41 +601,6 @@ class Foo(object): assert ob.ObjectField == Foo -def test_enum_conversion(): - """Test enum conversion.""" - from Python.Test import ShortEnum - - ob = ConversionTest() - assert ob.EnumField == ShortEnum.Zero - - ob.EnumField = ShortEnum.One - assert ob.EnumField == ShortEnum.One - - ob.EnumField = 0 - assert ob.EnumField == ShortEnum.Zero - assert ob.EnumField == 0 - - ob.EnumField = 1 - assert ob.EnumField == ShortEnum.One - assert ob.EnumField == 1 - - with pytest.raises(ValueError): - ob = ConversionTest() - ob.EnumField = 10 - - with pytest.raises(ValueError): - ob = ConversionTest() - ob.EnumField = 255 - - with pytest.raises(OverflowError): - ob = ConversionTest() - ob.EnumField = 1000000 - - with pytest.raises(TypeError): - ob = ConversionTest() - ob.EnumField = "spam" - - def test_null_conversion(): """Test null conversion.""" import System diff --git a/src/tests/test_delegate.py b/tests/test_delegate.py similarity index 61% rename from src/tests/test_delegate.py rename to tests/test_delegate.py index 909fd0f05..52ac8226d 100644 --- a/src/tests/test_delegate.py +++ b/tests/test_delegate.py @@ -276,6 +276,166 @@ def test_invalid_object_delegate(): with pytest.raises(TypeError): ob.CallObjectDelegate(d) +def test_out_int_delegate(): + """Test delegate with an out int parameter.""" + from Python.Test import OutIntDelegate + value = 7 + + def out_hello_func(ignored): + return 5 + + d = OutIntDelegate(out_hello_func) + result = d(value) + assert result == 5 + + ob = DelegateTest() + result = ob.CallOutIntDelegate(d, value) + assert result == 5 + + def invalid_handler(ignored): + return '5' + + d = OutIntDelegate(invalid_handler) + with pytest.raises(TypeError): + result = d(value) + +def test_out_string_delegate(): + """Test delegate with an out string parameter.""" + from Python.Test import OutStringDelegate + value = 'goodbye' + + def out_hello_func(ignored): + return 'hello' + + d = OutStringDelegate(out_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallOutStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefIntDelegate + value = 7 + + def ref_hello_func(data): + assert data == value + return data + 1 + + d = RefIntDelegate(ref_hello_func) + result = d(value) + assert result == value + 1 + + ob = DelegateTest() + result = ob.CallRefIntDelegate(d, value) + assert result == value + 1 + +def test_ref_string_delegate(): + """Test delegate with a ref string parameter.""" + from Python.Test import RefStringDelegate + value = 'goodbye' + + def ref_hello_func(data): + assert data == value + return 'hello' + + d = RefStringDelegate(ref_hello_func) + result = d(value) + assert result == 'hello' + + ob = DelegateTest() + result = ob.CallRefStringDelegate(d, value) + assert result == 'hello' + +def test_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import RefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + 1, stringValue + '!') + + d = RefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = RefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = RefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def long_tuple(intValue, stringValue): + return (5, 'a', 'b') + + d = RefIntRefStringDelegate(long_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_tuple_item(intValue, stringValue): + return ('a', 'b') + + d = RefIntRefStringDelegate(wrong_tuple_item) + with pytest.raises(TypeError): + result = d(intData, stringData) + +def test_int_ref_int_ref_string_delegate(): + """Test delegate with a ref int and ref string parameter.""" + from Python.Test import IntRefIntRefStringDelegate + intData = 7 + stringData = 'goodbye' + + def ref_hello_func(intValue, stringValue): + assert intData == intValue + assert stringData == stringValue + return (intValue + len(stringValue), intValue + 1, stringValue + '!') + + d = IntRefIntRefStringDelegate(ref_hello_func) + result = d(intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + ob = DelegateTest() + result = ob.CallIntRefIntRefStringDelegate(d, intData, stringData) + assert result == (intData + len(stringData), intData + 1, stringData + '!') + + def not_a_tuple(intValue, stringValue): + return 'a' + + d = IntRefIntRefStringDelegate(not_a_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def short_tuple(intValue, stringValue): + return (5,) + + d = IntRefIntRefStringDelegate(short_tuple) + with pytest.raises(TypeError): + result = d(intData, stringData) + + def wrong_return_type(intValue, stringValue): + return ('a', 7, 'b') + + d = IntRefIntRefStringDelegate(wrong_return_type) + with pytest.raises(TypeError): + result = d(intData, stringData) + # test async delegates # test multicast delegates diff --git a/src/tests/test_docstring.py b/tests/test_docstring.py similarity index 100% rename from src/tests/test_docstring.py rename to tests/test_docstring.py diff --git a/src/tests/test_engine.py b/tests/test_engine.py similarity index 100% rename from src/tests/test_engine.py rename to tests/test_engine.py diff --git a/src/tests/test_enum.py b/tests/test_enum.py similarity index 51% rename from src/tests/test_enum.py rename to tests/test_enum.py index 27fe7e9ef..1f0711a94 100644 --- a/src/tests/test_enum.py +++ b/tests/test_enum.py @@ -22,69 +22,69 @@ def test_enum_get_member(): """Test access to enum members.""" from System import DayOfWeek - assert DayOfWeek.Sunday == 0 - assert DayOfWeek.Monday == 1 - assert DayOfWeek.Tuesday == 2 - assert DayOfWeek.Wednesday == 3 - assert DayOfWeek.Thursday == 4 - assert DayOfWeek.Friday == 5 - assert DayOfWeek.Saturday == 6 + assert DayOfWeek.Sunday == DayOfWeek(0) + assert DayOfWeek.Monday == DayOfWeek(1) + assert DayOfWeek.Tuesday == DayOfWeek(2) + assert DayOfWeek.Wednesday == DayOfWeek(3) + assert DayOfWeek.Thursday == DayOfWeek(4) + assert DayOfWeek.Friday == DayOfWeek(5) + assert DayOfWeek.Saturday == DayOfWeek(6) def test_byte_enum(): """Test byte enum.""" - assert Test.ByteEnum.Zero == 0 - assert Test.ByteEnum.One == 1 - assert Test.ByteEnum.Two == 2 + assert Test.ByteEnum.Zero == Test.ByteEnum(0) + assert Test.ByteEnum.One == Test.ByteEnum(1) + assert Test.ByteEnum.Two == Test.ByteEnum(2) def test_sbyte_enum(): """Test sbyte enum.""" - assert Test.SByteEnum.Zero == 0 - assert Test.SByteEnum.One == 1 - assert Test.SByteEnum.Two == 2 + assert Test.SByteEnum.Zero == Test.SByteEnum(0) + assert Test.SByteEnum.One == Test.SByteEnum(1) + assert Test.SByteEnum.Two == Test.SByteEnum(2) def test_short_enum(): """Test short enum.""" - assert Test.ShortEnum.Zero == 0 - assert Test.ShortEnum.One == 1 - assert Test.ShortEnum.Two == 2 + assert Test.ShortEnum.Zero == Test.ShortEnum(0) + assert Test.ShortEnum.One == Test.ShortEnum(1) + assert Test.ShortEnum.Two == Test.ShortEnum(2) def test_ushort_enum(): """Test ushort enum.""" - assert Test.UShortEnum.Zero == 0 - assert Test.UShortEnum.One == 1 - assert Test.UShortEnum.Two == 2 + assert Test.UShortEnum.Zero == Test.UShortEnum(0) + assert Test.UShortEnum.One == Test.UShortEnum(1) + assert Test.UShortEnum.Two == Test.UShortEnum(2) def test_int_enum(): """Test int enum.""" - assert Test.IntEnum.Zero == 0 - assert Test.IntEnum.One == 1 - assert Test.IntEnum.Two == 2 + assert Test.IntEnum.Zero == Test.IntEnum(0) + assert Test.IntEnum.One == Test.IntEnum(1) + assert Test.IntEnum.Two == Test.IntEnum(2) def test_uint_enum(): """Test uint enum.""" - assert Test.UIntEnum.Zero == 0 - assert Test.UIntEnum.One == 1 - assert Test.UIntEnum.Two == 2 + assert Test.UIntEnum.Zero == Test.UIntEnum(0) + assert Test.UIntEnum.One == Test.UIntEnum(1) + assert Test.UIntEnum.Two == Test.UIntEnum(2) def test_long_enum(): """Test long enum.""" - assert Test.LongEnum.Zero == 0 - assert Test.LongEnum.One == 1 - assert Test.LongEnum.Two == 2 + assert Test.LongEnum.Zero == Test.LongEnum(0) + assert Test.LongEnum.One == Test.LongEnum(1) + assert Test.LongEnum.Two == Test.LongEnum(2) def test_ulong_enum(): """Test ulong enum.""" - assert Test.ULongEnum.Zero == 0 - assert Test.ULongEnum.One == 1 - assert Test.ULongEnum.Two == 2 + assert Test.ULongEnum.Zero == Test.ULongEnum(0) + assert Test.ULongEnum.One == Test.ULongEnum(1) + assert Test.ULongEnum.Two == Test.ULongEnum(2) def test_instantiate_enum_fails(): @@ -117,29 +117,31 @@ def test_enum_set_member_fails(): del DayOfWeek.Sunday -def test_enum_with_flags_attr_conversion(): +def test_enum_undefined_value(): """Test enumeration conversion with FlagsAttribute set.""" # This works because the FlagsField enum has FlagsAttribute. - Test.FieldTest().FlagsField = 99 + Test.FieldTest().FlagsField = Test.FlagsEnum(99) # This should fail because our test enum doesn't have it. with pytest.raises(ValueError): - Test.FieldTest().EnumField = 99 - + Test.FieldTest().EnumField = Test.ShortEnum(20) + + # explicitly permit undefined values + Test.FieldTest().EnumField = Test.ShortEnum(20, True) def test_enum_conversion(): """Test enumeration conversion.""" ob = Test.FieldTest() - assert ob.EnumField == 0 + assert ob.EnumField == Test.ShortEnum(0) ob.EnumField = Test.ShortEnum.One - assert ob.EnumField == 1 - - with pytest.raises(ValueError): - Test.FieldTest().EnumField = 20 + assert ob.EnumField == Test.ShortEnum(1) with pytest.raises(OverflowError): - Test.FieldTest().EnumField = 100000 + Test.FieldTest().EnumField = Test.ShortEnum(100000) with pytest.raises(TypeError): Test.FieldTest().EnumField = "str" + + with pytest.raises(TypeError): + Test.FieldTest().EnumField = 1 diff --git a/src/tests/test_event.py b/tests/test_event.py similarity index 93% rename from src/tests/test_event.py rename to tests/test_event.py index e9c0ffd8a..885589032 100644 --- a/src/tests/test_event.py +++ b/tests/test_event.py @@ -295,6 +295,41 @@ def handler(sender, args, dict_=dict_): ob.OnPublicEvent(EventArgsTest(20)) assert dict_['value'] == 10 +def test_out_function_handler(): + """Test function handlers with Out arguments.""" + ob = EventTest() + + value = 10 + def handler(ignored): + return value + + ob.OutIntEvent += handler + result = ob.OnOutIntEvent(55) + assert result == value + + ob.OutStringEvent += handler + value = 'This is the event data' + result = ob.OnOutStringEvent('Hello') + assert result == value + +def test_ref_function_handler(): + """Test function handlers with Ref arguments.""" + ob = EventTest() + + value = 10 + def handler(data): + return value + data + + ob.RefIntEvent += ob.RefIntHandler + ob.RefIntEvent += handler + result = ob.OnRefIntEvent(5) + assert result == value + 5 + 1 + + ob.RefStringEvent += ob.RefStringHandler + ob.RefStringEvent += handler + value = 'This is the event data' + result = ob.OnRefStringEvent('!') + assert result == value + '!!' def test_add_non_callable_handler(): """Test handling of attempts to add non-callable handlers.""" diff --git a/src/tests/test_exceptions.py b/tests/test_exceptions.py similarity index 100% rename from src/tests/test_exceptions.py rename to tests/test_exceptions.py diff --git a/src/tests/test_field.py b/tests/test_field.py similarity index 100% rename from src/tests/test_field.py rename to tests/test_field.py diff --git a/src/tests/test_generic.py b/tests/test_generic.py similarity index 95% rename from src/tests/test_generic.py rename to tests/test_generic.py index c865ab32f..248303179 100644 --- a/src/tests/test_generic.py +++ b/tests/test_generic.py @@ -298,9 +298,8 @@ def test_generic_method_type_handling(): from Python.Test import InterfaceTest, ISayHello1, ShortEnum import System - # FIXME: The value doesn't fit into Int64 and PythonNet doesn't - # recognize it as UInt64 for unknown reasons. - # assert_generic_method_by_type(System.UInt64, 18446744073709551615L) + # FIXME: Fails because Converter.GetTypeByAlias returns int32 for any integer, including ulong + # assert_generic_method_by_type(System.UInt64, 18446744073709551615) assert_generic_method_by_type(System.Boolean, True) assert_generic_method_by_type(bool, True) assert_generic_method_by_type(System.Byte, 255) @@ -750,3 +749,32 @@ def test_missing_generic_type(): from System.Collections import IList with pytest.raises(TypeError): IList[bool] + +def test_invalid_generic_type_parameter(): + from Python.Test import GenericTypeWithConstraint + with pytest.raises(TypeError): + GenericTypeWithConstraint[System.Object] + +def test_invalid_generic_method_type_parameter(): + from Python.Test import ConversionTest + with pytest.raises(TypeError): + ConversionTest.Echo[System.Object] + +def test_generic_list_array_conversion(): + """Test conversion of lists to generic array arguments.""" + from Python.Test import GenericArrayConversionTest + from Python.Test import Spam + + items = [] + for i in range(10): + items.append(Spam(str(i))) + + result = GenericArrayConversionTest.EchoRange[Spam](items) + assert result[0].__class__ == Spam + assert len(result) == 10 + + # Currently raises an exception because the correct generic type parameter is not inferred + with pytest.raises(TypeError): + result = GenericArrayConversionTest.EchoRange(items) + assert result[0].__class__ == Spam + assert len(result) == 10 diff --git a/src/tests/test_import.py b/tests/test_import.py similarity index 100% rename from src/tests/test_import.py rename to tests/test_import.py diff --git a/src/tests/test_indexer.py b/tests/test_indexer.py similarity index 96% rename from src/tests/test_indexer.py rename to tests/test_indexer.py index b8a33fb46..0af6e6c45 100644 --- a/src/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -400,8 +400,10 @@ def test_enum_indexer(): ob[key] = "eggs" assert ob[key] == "eggs" - ob[1] = "spam" - assert ob[1] == "spam" + with pytest.raises(TypeError): + ob[1] = "spam" + with pytest.raises(TypeError): + ob[1] with pytest.raises(TypeError): ob = Test.EnumIndexerTest() @@ -646,3 +648,21 @@ def test_inherited_indexer_interface(): ifc = IInheritedIndexer(impl) ifc[0] = "zero" assert ifc[0] == "zero" + +def test_public_inherited_overloaded_indexer(): + """Test public indexers.""" + ob = Test.PublicInheritedOverloadedIndexer() + + ob[0] = "zero" + assert ob[0] == "zero" + + ob[1] = "one" + assert ob[1] == "one" + + assert ob[10] is None + + ob["spam"] = "spam" + assert ob["spam"] == "spam" + + with pytest.raises(TypeError): + ob[[]] diff --git a/src/tests/test_interface.py b/tests/test_interface.py similarity index 100% rename from src/tests/test_interface.py rename to tests/test_interface.py diff --git a/src/tests/test_method.py b/tests/test_method.py similarity index 97% rename from src/tests/test_method.py rename to tests/test_method.py index 18eb5af8e..9bdb571c0 100644 --- a/src/tests/test_method.py +++ b/tests/test_method.py @@ -807,6 +807,12 @@ def test_no_object_in_param(): with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") + with pytest.raises(TypeError): + MethodTest.TestOverloadedNoObject(5.5) + + # Ensure that the top-level error is TypeError even if the inner error is an OverflowError + with pytest.raises(TypeError): + MethodTest.TestOverloadedNoObject(2147483648) def test_object_in_param(): """Test regression introduced by #151 in which Object method overloads @@ -886,9 +892,13 @@ def test_object_in_multiparam(): def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" - with pytest.raises(TypeError): + with pytest.raises(TypeError) as excinfo: MethodTest.TestOverloadedObjectThree("foo", "bar") + e = excinfo.value + c = e.__cause__ + assert c.GetType().FullName == 'System.AggregateException' + assert len(c.InnerExceptions) == 2 def test_case_sensitive(): """Test that case-sensitivity is respected. GH#81""" @@ -1213,13 +1223,17 @@ def test_params_array_overload(): res = MethodTest.ParamsArrayOverloaded(1, 2, 3, i=1) assert res == "with params-array" - # These two cases are still incorrectly failing: - - # res = MethodTest.ParamsArrayOverloaded(1, 2, i=1) - # assert res == "with params-array" +@pytest.mark.skip(reason="FIXME: incorrectly failing") +def test_params_array_overloaded_failing(): + res = MethodTest.ParamsArrayOverloaded(1, 2, i=1) + assert res == "with params-array" - # res = MethodTest.ParamsArrayOverloaded(paramsArray=[], i=1) - # assert res == "with params-array" + res = MethodTest.ParamsArrayOverloaded(paramsArray=[], i=1) + assert res == "with params-array" def test_method_encoding(): MethodTest.EncodingTestÅngström() + +def test_method_with_pointer_array_argument(): + with pytest.raises(TypeError): + MethodTest.PointerArray([0]) diff --git a/src/tests/test_module.py b/tests/test_module.py similarity index 92% rename from src/tests/test_module.py rename to tests/test_module.py index dcdb0248e..d0378e91e 100644 --- a/src/tests/test_module.py +++ b/tests/test_module.py @@ -30,7 +30,7 @@ def test_import_clr(): def test_version_clr(): import clr - assert clr.__version__ >= "2.2.0" + assert clr.__version__ >= "3.0.0" def test_preload_var(): @@ -111,12 +111,13 @@ def test_dotted_name_import(): def test_multiple_dotted_name_import(): """Test an import bug with multiple dotted imports.""" - import System.Data - assert is_clr_module(System.Data) - assert System.Data.__name__ == 'System.Data' - import System.Data - assert is_clr_module(System.Data) - assert System.Data.__name__ == 'System.Data' + + import System.Reflection + assert is_clr_module(System.Reflection) + assert System.Reflection.__name__ == 'System.Reflection' + import System.Reflection + assert is_clr_module(System.Reflection) + assert System.Reflection.__name__ == 'System.Reflection' def test_dotted_name_import_with_alias(): @@ -192,11 +193,11 @@ def test_dotted_name_import_from_with_alias(): def test_from_module_import_star(): """Test from module import * behavior.""" - clr.AddReference('System.Xml') - + clr.AddReference("System") + count = len(locals().keys()) - m = __import__('System.Xml', globals(), locals(), ['*']) - assert m.__name__ == 'System.Xml' + m = __import__('System', globals(), locals(), ['*']) + assert m.__name__ == 'System' assert is_clr_module(m) assert len(locals().keys()) > count + 1 @@ -212,7 +213,11 @@ def test_implicit_assembly_load(): import Microsoft.Build with warnings.catch_warnings(record=True) as w: - clr.AddReference("System.Windows.Forms") + try: + clr.AddReference("System.Windows.Forms") + except Exception: + pytest.skip() + import System.Windows.Forms as Forms assert is_clr_module(Forms) assert Forms.__name__ == 'System.Windows.Forms' @@ -227,11 +232,11 @@ def test_explicit_assembly_load(): from System.Reflection import Assembly import System, sys - assembly = Assembly.LoadWithPartialName('System.Data') + assembly = Assembly.LoadWithPartialName('System.Runtime') assert assembly is not None - import System.Data - assert 'System.Data' in sys.modules + import System.Runtime + assert 'System.Runtime' in sys.modules assembly = Assembly.LoadWithPartialName('SpamSpamSpamSpamEggsAndSpam') assert assembly is None @@ -275,12 +280,14 @@ def test_module_lookup_recursion(): def test_module_get_attr(): """Test module getattr behavior.""" + import System + import System.Runtime int_type = System.Int32 assert is_clr_class(int_type) - module = System.Xml + module = System.Runtime assert is_clr_module(module) with pytest.raises(AttributeError): @@ -324,7 +331,6 @@ def test_clr_list_assemblies(): from clr import ListAssemblies verbose = list(ListAssemblies(True)) short = list(ListAssemblies(False)) - assert u'mscorlib' in short assert u'System' in short assert u'Culture=' in verbose[0] assert u'Version=' in verbose[0] @@ -377,8 +383,11 @@ def test_assembly_load_thread_safety(): _ = Dictionary[Guid, DateTime]() ModuleTest.JoinThreads() +@pytest.mark.skipif() def test_assembly_load_recursion_bug(): """Test fix for recursion bug documented in #627""" - from System.Configuration import ConfigurationManager - content = dir(ConfigurationManager) + sys_config = pytest.importorskip( + "System.Configuration", reason="System.Configuration can't be imported" + ) + content = dir(sys_config.ConfigurationManager) assert len(content) > 5, content diff --git a/src/tests/test_mp_length.py b/tests/test_mp_length.py similarity index 100% rename from src/tests/test_mp_length.py rename to tests/test_mp_length.py diff --git a/src/tests/test_property.py b/tests/test_property.py similarity index 100% rename from src/tests/test_property.py rename to tests/test_property.py diff --git a/src/tests/test_recursive_types.py b/tests/test_recursive_types.py similarity index 100% rename from src/tests/test_recursive_types.py rename to tests/test_recursive_types.py diff --git a/src/tests/test_repr.py b/tests/test_repr.py similarity index 100% rename from src/tests/test_repr.py rename to tests/test_repr.py diff --git a/src/tests/test_subclass.py b/tests/test_subclass.py similarity index 94% rename from src/tests/test_subclass.py rename to tests/test_subclass.py index 968f6a788..4f3180480 100644 --- a/src/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -57,6 +57,14 @@ def return_list(self): return DerivedClass +def broken_derived_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class DerivedClass(SubClassTest): + """class that derives from a class deriving from IInterfaceTest""" + __namespace__ = 3 + + return DerivedClass def derived_event_test_class_fixture(subnamespace): """Delay creation of class until test starts.""" @@ -127,6 +135,11 @@ def test_derived_class(): x = FunctionsTest.pass_through(ob) assert id(x) == id(ob) +def test_broken_derived_class(): + """Test python class derived from managed type with invalid namespace""" + with pytest.raises(TypeError): + DerivedClass = broken_derived_class_fixture(test_derived_class.__name__) + ob = DerivedClass() def test_derived_traceback(): """Test python exception traceback in class derived from managed base""" diff --git a/src/tests/test_sysargv.py b/tests/test_sysargv.py similarity index 77% rename from src/tests/test_sysargv.py rename to tests/test_sysargv.py index dd62bc632..d856ec902 100644 --- a/src/tests/test_sysargv.py +++ b/tests/test_sysargv.py @@ -2,6 +2,7 @@ import sys from subprocess import check_output +from ast import literal_eval def test_sys_argv_state(filepath): @@ -11,5 +12,5 @@ def test_sys_argv_state(filepath): script = filepath("argv-fixture.py") out = check_output([sys.executable, script, "foo", "bar"]) - assert b"foo" in out - assert b"bar" in out + out = literal_eval(out.decode("ascii")) + assert out[-2:] == ["foo", "bar"] diff --git a/src/tests/test_thread.py b/tests/test_thread.py similarity index 100% rename from src/tests/test_thread.py rename to tests/test_thread.py diff --git a/src/tests/tests.pyproj b/tests/tests.pyproj similarity index 83% rename from src/tests/tests.pyproj rename to tests/tests.pyproj index 4bdbc6b14..fc1053f7b 100644 --- a/src/tests/tests.pyproj +++ b/tests/tests.pyproj @@ -14,14 +14,8 @@ - - - - - - - - + + 10.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets @@ -58,6 +52,7 @@ + diff --git a/src/tests/utils.py b/tests/utils.py similarity index 100% rename from src/tests/utils.py rename to tests/utils.py diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 0d5b83b30..0c80c1904 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -225,8 +225,6 @@ def preprocess_python_headers(): if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) - if "m" in sys.abiflags: - defines.extend(("-D", "PYTHON_WITH_PYMALLOC")) if "u" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) @@ -245,7 +243,7 @@ def preprocess_python_headers(): def gen_interop_head(writer): filename = os.path.basename(__file__) - abi_flags = getattr(sys, "abiflags", "") + abi_flags = getattr(sys, "abiflags", "").replace("m", "") py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) class_definition = """ // Auto-generated by %s.