diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index ac1d31324..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,29 +0,0 @@ -[bumpversion] -current_version = 2.4.0.dev0 -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}.{release}{dev} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = dummy -values = - dev - dummy - -[bumpversion:part:dev] - -[bumpversion:file:setup.py] - -[bumpversion:file:conda.recipe/meta.yaml] - -[bumpversion:file:src/runtime/resources/clr.py] - -[bumpversion:file:src/SharedAssemblyInfo.cs] -serialize = - {major}.{minor}.{patch} - -[bumpversion:file:src/clrmodule/ClrModule.cs] -serialize = - {major}.{minor}.{patch} - diff --git a/.editorconfig b/.editorconfig index 2e7c58ffe..d64f74bc1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,6 +19,17 @@ indent_size = 2 [*.{csproj,pyproj,config}] indent_size = 2 +# .NET formatting settings +[*.{cs,vb}] +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = true + +[*.cs] +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true + # Solution [*.sln] indent_style = tab diff --git a/.gitignore b/.gitignore index 6f813dcb0..1e494b12d 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,10 @@ UpgradeLog*.htm # Coverity cov-int/ + +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json diff --git a/.travis.yml b/.travis.yml index d059bdcde..5a2c975eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,87 +1,17 @@ -dist: trusty +dist: xenial sudo: false language: python - -matrix: - include: -# --------------------- XPLAT builds ------------------------ - - python: 2.7 - env: &xplat-env - - BUILD_OPTS=--xplat - - NUNIT_PATH=~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe - addons: &xplat-addons - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-trusty-prod trusty main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 - - - python: 3.4 - env: *xplat-env - addons: *xplat-addons - - - python: 3.5 - env: *xplat-env - addons: *xplat-addons - - - python: 3.6 - env: *xplat-env - addons: *xplat-addons - - - python: 3.7 - env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 - -# --------------------- Classic builds ------------------------ - - python: 2.7 - env: &classic-env - - BUILD_OPTS= - - NUNIT_PATH=./packages/NUnit.*/tools/nunit3-console.exe - - - python: 3.4 - env: *classic-env - - - python: 3.5 - env: *classic-env - - - python: 3.6 - env: *classic-env - - - python: 3.7 - env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono +python: + - 3.8 + - 3.7 + - 3.6 + - 3.5 env: + matrix: + - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" + global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - SEGFAULT_SIGNALS=all @@ -91,11 +21,16 @@ env: addons: apt: sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main + - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main + key_url: https://packages.microsoft.com/keys/microsoft.asc + - sourceline: deb http://download.mono-project.com/repo/ubuntu stable-xenial/snapshots/5.20 main key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF packages: - mono-devel - ca-certificates-mono + - dotnet-hostfxr-2.2 + - dotnet-runtime-2.2 + - dotnet-sdk-2.2 before_install: # Set-up dll path for embedded tests @@ -109,13 +44,11 @@ install: script: - python -m pytest - - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.dll - - if [[ $BUILD_OPTS == --xplat ]]; then dotnet src/embed_tests/bin/netcoreapp2.0_publish/Python.EmbeddingTest.dll; fi + - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll --labels=All + # does not work on Linux, because NuGet package for 2.3 is Windows only + # - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi" after_script: - # Uncomment if need to geninterop, ie. py37 final - # - python tools/geninterop/geninterop.py - # Waiting on mono-coverage, SharpCover or xr.Baboon - coverage xml -i - codecov --file coverage.xml --flags setup_linux diff --git a/AUTHORS.md b/AUTHORS.md index fe2d2b172..85812a47d 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,8 +2,11 @@ ## Development Lead -- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Benedikt Reinartz ([@filmor](https://github.com/filmor)) +- Victor Milovanov ([@lostmsu](https://github.com/lostmsu)) + +## Former Development Leads +- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Brian Lloyd ([@brianlloyd](https://github.com/brianlloyd)) - David Anthoff ([@davidanthoff](https://github.com/davidanthoff)) - Denis Akhiyarov ([@denfromufa](https://github.com/denfromufa)) @@ -12,13 +15,17 @@ ## Contributors +- Alex Earl ([@slide](https://github.com/slide)) +- Alex Helms ([@alexhelms](https://github.com/alexhelms)) - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) +- Andrey Sant'Anna ([@andreydani](https://github.com/andreydani)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) - Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) +- Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) - Christopher Pow ([@christopherpow](https://github.com/christopherpow)) - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) @@ -26,16 +33,23 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) --   Ivan Cronyn ([@cronan](https://github.com/cronan)) +- Inna Wiesel ([@inna-w](https://github.com/inna-w)) +- Ivan Cronyn ([@cronan](https://github.com/cronan)) - Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) --   Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Robbins ([@jeff17robbins](https://github.com/jeff17robbins)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) +- Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) +- Joe Savage ([@s4v4g3](https://github.com/s4v4g3)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) - John Wilkes ([@jbw3](https://github.com/jbw3)) - Luke Stratman ([@lstratman](https://github.com/lstratman)) - Konstantin Posudevskiy ([@konstantin-posudevskiy](https://github.com/konstantin-posudevskiy)) - Matthias Dittrich ([@matthid](https://github.com/matthid)) +- Meinrad Recheis ([@henon](https://github.com/henon)) +- Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) @@ -54,8 +68,10 @@ - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) - ([@omnicognate](https://github.com/omnicognate)) +- ([@OneBlue](https://github.com/OneBlue)) - ([@rico-chet](https://github.com/rico-chet)) - ([@rmadsen-ks](https://github.com/rmadsen-ks)) - ([@stonebig](https://github.com/stonebig)) - ([@testrunner123](https://github.com/testrunner123)) +- ([@DanBarzilian](https://github.com/DanBarzilian)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3816cd0c..69b4c3e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,75 @@ # Changelog -All notable changes to Python for .NET will be documented in this file. -This project adheres to [Semantic Versioning][]. +All notable changes to Python.NET will be documented in this file. This +project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. -## [unreleased][] +## [Unreleased][] + +### Added + +### Changed +- Drop support for Python 2 + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling + +## [2.5.0][] - 2020-06-14 + +This version improves performance on benchmarks significantly compared to 2.3. + +### Added + +- Automatic NuGet package generation in appveyor and local builds +- Function that sets `Py_NoSiteFlag` to 1. +- Support for Jetson Nano. +- Support for `__len__` for .NET classes that implement ICollection +- `PyExport` attribute to hide .NET types from Python +- `PythonException.Format` method to format exceptions the same as + `traceback.format_exception` +- `Runtime.None` to be able to pass `None` as parameter into Python from .NET +- `PyObject.IsNone()` to check if a Python object is None in .NET. +- Support for Python 3.8 +- Codecs as the designated way to handle automatic conversions between + .NET and Python types + +### Changed + +- Added argument types information to "No method matches given arguments" message +- Moved wheel import in setup.py inside of a try/except to prevent pip collection failures +- Removes `PyLong_GetMax` and `PyClass_New` when targetting Python3 +- Improved performance of calls from Python to C# +- Added support for converting python iterators to C# arrays +- Changed usage of the obsolete function + `GetDelegateForFunctionPointer(IntPtr, Type)` to + `GetDelegateForFunctionPointer(IntPtr)` +- When calling C# from Python, enable passing argument of any type to a + parameter of C# type `object` by wrapping it into `PyObject` instance. + ([#881][i881]) +- Added support for kwarg parameters when calling .NET methods from Python +- Changed method for finding MSBuild using vswhere +- Reworked `Finalizer`. Now objects drop into its queue upon finalization, + which is periodically drained when new objects are created. +- Marked `Runtime.OperatingSystemName` and `Runtime.MachineName` as + `Obsolete`, should never have been `public` in the first place. They also + don't necessarily return a result that matches the `platform` module's. +- Unconditionally depend on `pycparser` for the interop module generation + +### Fixed + +- Fixed runtime that fails loading when using pythonnet in an environment + together with Nuitka +- Fixes bug where delegates get casts (dotnetcore) +- Determine size of interpreter longs at runtime +- Handling exceptions ocurred in ModuleObject's getattribute +- Fill `__classcell__` correctly for Python subclasses of .NET types +- Fixed issue with params methods that are not passed an array. +- Use UTF8 to encode strings passed to `PyRun_String` on Python 3 + +## [2.4.0][] - 2019-05-15 ### Added @@ -21,13 +85,19 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now ([#692][p692]). - Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) - Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added support for C# types to provide `__repr__` ([#680][p680]) ### Changed +- PythonException included C# call stack - Reattach python exception traceback information (#545) - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) +- Refactored MethodBinder.Bind in preparation to make it extensible (#829) +- Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- improved performance of calls from Python to C# ### Fixed @@ -43,7 +113,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed conversion of 'float' and 'double' values ([#486][i486]) - Fixed 'clrmethod' for python 2 ([#492][i492]) - Fixed double calling of constructor when deriving from .NET class ([#495][i495]) -- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) - Fixed `LockRecursionException` when loading assemblies ([#627][i627]) - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea0f3408e..ffeb792f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,15 @@ # How to contribute -PythonNet is developed and maintained by unpaid community members so well +Python.NET is developed and maintained by unpaid community members so well written, documented and tested pull requests are encouraged. By submitting a pull request for this project, you agree to license your contribution under the MIT license to this project. +This project has adopted the code of conduct defined by the Contributor +Covenant to clarify expected behavior in our community. For more information +see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). + ## Getting Started - Make sure you have a [GitHub account](https://github.com/signup/free) @@ -41,3 +45,4 @@ contribution under the MIT license to this project. - [General GitHub documentation](https://help.github.com/) - [GitHub pull request documentation](https://help.github.com/send-pull-requests/) +- [.NET Foundation Code of Conduct](https://dotnetfoundation.org/about/code-of-conduct) diff --git a/LICENSE b/LICENSE index e344a0795..19c31a12f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2006-2017 the contributors of the "Python for .NET" project +Copyright (c) 2006-2020 the contributors of the Python.NET project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), diff --git a/Package.nuspec b/Package.nuspec new file mode 100644 index 000000000..0d49a8706 --- /dev/null +++ b/Package.nuspec @@ -0,0 +1,28 @@ + + + + QuantConnect.pythonnet + 1.0.5.30 + QuantConnect Python Team + QuantConnect Corp. + http://www.QuantConnect.com + http://www.QuantConnect.com + false + QuantConnect PythonNET package + QuantConnect PythonNET package + Copyright 2020 + QuantConnect PythonNET + + + + + + + + + + + + + + \ No newline at end of file diff --git a/QuantConnect.pythonnet.targets b/QuantConnect.pythonnet.targets new file mode 100644 index 000000000..95382c500 --- /dev/null +++ b/QuantConnect.pythonnet.targets @@ -0,0 +1,8 @@ + + + + nPython.exe + Always + + + \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 7e859481d..000000000 --- a/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# pythonnet - Python for .NET - -[![Join the chat at https://gitter.im/pythonnet/pythonnet](https://badges.gitter.im/pythonnet/pythonnet.svg)](https://gitter.im/pythonnet/pythonnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -[![appveyor shield][]](https://ci.appveyor.com/project/pythonnet/pythonnet/branch/master) -[![travis shield][]](https://travis-ci.org/pythonnet/pythonnet) -[![codecov shield][]](https://codecov.io/github/pythonnet/pythonnet) -[![coverity shield][]](https://scan.coverity.com/projects/pythonnet) - -[![license shield][]](./LICENSE) -[![pypi package version][]](https://pypi.python.org/pypi/pythonnet) -[![python supported shield][]](https://pypi.python.org/pypi/pythonnet) -[![stackexchange shield][]](http://stackoverflow.com/questions/tagged/python.net) -[![slack][]](https://pythonnet.slack.com) - -Python for .NET is a package that gives Python programmers nearly -seamless integration with the .NET Common Language Runtime (CLR) and -provides a powerful application scripting tool for .NET developers. -It allows Python code to interact with the CLR, and may also be used to -embed Python into a .NET application. - -## Calling .NET code from Python - -Python for .NET allows CLR namespaces to be treated essentially -as Python packages. - -```python -import clr -from System import String -from System.Collections import * -``` - -To load an assembly, use the `AddReference` function in the `clr` module: - -```python -import clr -clr.AddReference("System.Windows.Forms") -from System.Windows.Forms import Form -``` - -## Embedding Python in .NET - -- All calls to python should be inside - a `using (Py.GIL()) {/* Your code here */}` block. -- Import python modules using `dynamic mod = Py.Import("mod")`, - then you can call functions as normal, eg `mod.func(args)`. -- Use `mod.func(args, Py.kw("keywordargname", keywordargvalue))` or `mod.func(args, keywordargname: keywordargvalue)` - to apply keyword arguments. -- All python objects should be declared as `dynamic` type. -- Mathematical operations involving python and literal/managed types must - have the python object first, eg. `np.pi * 2` works, `2 * np.pi` doesn't. - -### Example - -```csharp -static void Main(string[] args) -{ - using (Py.GIL()) - { - dynamic np = Py.Import("numpy"); - Console.WriteLine(np.cos(np.pi * 2)); - - dynamic sin = np.sin; - Console.WriteLine(sin(5)); - - double c = np.cos(5) + sin(5); - Console.WriteLine(c); - - dynamic a = np.array(new List { 1, 2, 3 }); - Console.WriteLine(a.dtype); - - dynamic b = np.array(new List { 6, 5, 4 }, dtype: np.int32); - Console.WriteLine(b.dtype); - - Console.WriteLine(a * b); - Console.ReadKey(); - } -} -``` - -Output: - -```c -1.0 --0.958924274663 --0.6752620892 -float64 -int32 -[ 6. 10. 12.] -``` - -Information on installation, FAQ, troubleshooting, debugging, and projects using pythonnet can be found in the Wiki: - -https://github.com/pythonnet/pythonnet/wiki - -[appveyor shield]: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor - -[codecov shield]: https://img.shields.io/codecov/c/github/pythonnet/pythonnet/master.svg?label=Codecov - -[coverity shield]: https://img.shields.io/coverity/scan/7830.svg - -[license shield]: https://img.shields.io/badge/license-MIT-blue.svg?maxAge=3600 - -[pypi package version]: https://img.shields.io/pypi/v/pythonnet.svg - -[python supported shield]: https://img.shields.io/pypi/pyversions/pythonnet.svg - -[slack]: https://img.shields.io/badge/chat-slack-color.svg?style=social - -[stackexchange shield]: https://img.shields.io/badge/StackOverflow-python.net-blue.svg - -[travis shield]: https://img.shields.io/travis/pythonnet/pythonnet/master.svg?label=Travis diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..55f0e50a1 --- /dev/null +++ b/README.rst @@ -0,0 +1,128 @@ +pythonnet - Python.NET +=========================== + +|Join the chat at https://gitter.im/pythonnet/pythonnet| + +|appveyor shield| |travis shield| |codecov shield| + +|license shield| |pypi package version| |conda-forge version| |python supported shield| +|stackexchange shield| + +Python.NET is a package that gives Python programmers nearly +seamless integration with the .NET Common Language Runtime (CLR) and +provides a powerful application scripting tool for .NET developers. It +allows Python code to interact with the CLR, and may also be used to +embed Python into a .NET application. + +Calling .NET code from Python +----------------------------- + +Python.NET allows CLR namespaces to be treated essentially as Python packages. + +.. code-block:: + + import clr + from System import String + from System.Collections import * + +To load an assembly, use the ``AddReference`` function in the ``clr`` +module: + +.. code-block:: + + import clr + clr.AddReference("System.Windows.Forms") + from System.Windows.Forms import Form + +Embedding Python in .NET +------------------------ + +- All calls to python should be inside a + ``using (Py.GIL()) {/* Your code here */}`` block. +- Import python modules using ``dynamic mod = Py.Import("mod")``, then + you can call functions as normal, eg ``mod.func(args)``. +- Use ``mod.func(args, Py.kw("keywordargname", keywordargvalue))`` or + ``mod.func(args, keywordargname: keywordargvalue)`` to apply keyword + arguments. +- All python objects should be declared as ``dynamic`` type. +- Mathematical operations involving python and literal/managed types + must have the python object first, eg. ``np.pi * 2`` works, + ``2 * np.pi`` doesn't. + +Example +~~~~~~~ + +.. code-block:: csharp + + static void Main(string[] args) + { + using (Py.GIL()) + { + dynamic np = Py.Import("numpy"); + Console.WriteLine(np.cos(np.pi * 2)); + + dynamic sin = np.sin; + Console.WriteLine(sin(5)); + + double c = np.cos(5) + sin(5); + Console.WriteLine(c); + + dynamic a = np.array(new List { 1, 2, 3 }); + Console.WriteLine(a.dtype); + + dynamic b = np.array(new List { 6, 5, 4 }, dtype: np.int32); + Console.WriteLine(b.dtype); + + Console.WriteLine(a * b); + Console.ReadKey(); + } + } + +Output: + +.. code:: + + 1.0 + -0.958924274663 + -0.6752620892 + float64 + int32 + [ 6. 10. 12.] + + + +Resources +--------- + +Information on installation, FAQ, troubleshooting, debugging, and +projects using pythonnet can be found in the Wiki: + +https://github.com/pythonnet/pythonnet/wiki + +Mailing list + https://mail.python.org/mailman/listinfo/pythondotnet +Chat + https://gitter.im/pythonnet/pythonnet + +.NET Foundation +--------------- +This project is supported by the `.NET Foundation `_. + +.. |Join the chat at https://gitter.im/pythonnet/pythonnet| image:: https://badges.gitter.im/pythonnet/pythonnet.svg + :target: https://gitter.im/pythonnet/pythonnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |appveyor shield| image:: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor + :target: https://ci.appveyor.com/project/pythonnet/pythonnet/branch/master +.. |travis shield| image:: https://img.shields.io/travis/pythonnet/pythonnet/master.svg?label=Travis + :target: https://travis-ci.org/pythonnet/pythonnet +.. |codecov shield| image:: https://img.shields.io/codecov/c/github/pythonnet/pythonnet/master.svg?label=Codecov + :target: https://codecov.io/github/pythonnet/pythonnet +.. |license shield| image:: https://img.shields.io/badge/license-MIT-blue.svg?maxAge=3600 + :target: ./LICENSE +.. |pypi package version| image:: https://img.shields.io/pypi/v/pythonnet.svg + :target: https://pypi.python.org/pypi/pythonnet +.. |python supported shield| image:: https://img.shields.io/pypi/pyversions/pythonnet.svg + :target: https://pypi.python.org/pypi/pythonnet +.. |stackexchange shield| image:: https://img.shields.io/badge/StackOverflow-python.net-blue.svg + :target: http://stackoverflow.com/questions/tagged/python.net +.. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg + :target: https://anaconda.org/conda-forge/pythonnet diff --git a/appveyor.yml b/appveyor.yml index ce345cbf8..d353bbe5f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ build: off image: - Visual Studio 2017 - + platform: - x86 - x64 @@ -15,27 +15,18 @@ environment: CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: - - PYTHON_VERSION: 2.7 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 + - PYTHON_VERSION: 3.8 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 2.7 - - PYTHON_VERSION: 3.4 - PYTHON_VERSION: 3.5 - - PYTHON_VERSION: 3.6 + BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 + - PYTHON_VERSION: 3.6 + - PYTHON_VERSION: 3.5 init: # Update Environment Variables based on matrix/platform @@ -60,9 +51,8 @@ build_script: - coverage run setup.py bdist_wheel %BUILD_OPTS% test_script: - - pip install --no-index --find-links=.\dist\ pythonnet + - pip install --find-links=.\dist\ pythonnet - ps: .\ci\appveyor_run_tests.ps1 - - ps: .\ci\appveyor_build_recipe.ps1 on_finish: # Temporary disable multiple upload due to codecov limit of 20 per commit. @@ -75,6 +65,7 @@ on_finish: artifacts: - path: dist\* + - path: '.\src\runtime\bin\*.nupkg' notifications: - provider: Slack diff --git a/ci/appveyor_build_recipe.ps1 b/ci/appveyor_build_recipe.ps1 deleted file mode 100644 index 84e0bc7c6..000000000 --- a/ci/appveyor_build_recipe.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -# Build `conda.recipe` only if this is a Pull_Request. Saves time for CI. - -$env:CONDA_PY = "$env:PY_VER" -# Use pre-installed miniconda. Note that location differs if 64bit -$env:CONDA_BLD = "C:\miniconda36" - -if ($env:PLATFORM -eq "x86"){ - $env:CONDA_BLD_ARCH=32 -} else { - $env:CONDA_BLD_ARCH=64 - $env:CONDA_BLD = "$env:CONDA_BLD" + "-x64" -} - -if ($env:APPVEYOR_PULL_REQUEST_NUMBER -or $env:APPVEYOR_REPO_TAG_NAME -or $env:FORCE_CONDA_BUILD -eq "True") { - # Update PATH, and keep a copy to restore at end of this PowerShell script - $old_path = $env:path - $env:path = "$env:CONDA_BLD;$env:CONDA_BLD\Scripts;" + $env:path - - Write-Host "Starting conda install" -ForegroundColor "Green" - conda config --set always_yes True - conda config --set changeps1 False - conda config --set auto_update_conda False - conda install conda-build jinja2 anaconda-client --quiet - conda info - - # why `2>&1 | %{ "$_" }`? Redirect STDERR to STDOUT - # see: http://stackoverflow.com/a/20950421/5208670 - Write-Host "Starting conda build recipe" -ForegroundColor "Green" - conda build conda.recipe --quiet 2>&1 | %{ "$_" } - - $CONDA_PKG=(conda build conda.recipe --output) - Copy-Item $CONDA_PKG .\dist\ - Write-Host "Completed conda build recipe" -ForegroundColor "Green" - - # Restore PATH back to original - $env:path = $old_path -} else { - Write-Host "Skipping conda build recipe" -ForegroundColor "Green" -} diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index b45440fbe..bd90943d5 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -1,8 +1,13 @@ # Script to simplify AppVeyor configuration and resolve path to tools +$stopwatch = [Diagnostics.Stopwatch]::StartNew() +[array]$timings = @() + # Test Runner framework being used for embedded tests $CS_RUNNER = "nunit3-console" +$XPLAT = $env:BUILD_OPTS -eq "--xplat" + # Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3 if ($FALSE -and $env:PLATFORM -eq "x86"){ $CS_RUNNER = $CS_RUNNER + "-x86" @@ -11,7 +16,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){ # Executable paths for OpenCover # Note if OpenCover fails, it won't affect the exit codes. $OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ $CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe } else{ @@ -23,6 +28,17 @@ $PY = Get-Command python $CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" $RUNTIME_DIR = ".\src\runtime\bin\" +function ReportTime { + param([string] $action) + + $timeSpent = $stopwatch.Elapsed + $timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent} + Write-Host $action " in " $timeSpent -ForegroundColor "Green" + $stopwatch.Restart() +} + +ReportTime "Preparation done" + # Run python tests with C# coverage Write-Host ("Starting Python tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage ` @@ -31,20 +47,51 @@ Write-Host ("Starting Python tests") -ForegroundColor "Green" $PYTHON_STATUS = $LastExitCode if ($PYTHON_STATUS -ne 0) { Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Python tests completed" } # Run Embedded tests with C# coverage Write-Host ("Starting embedded tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage ` - -target:"$CS_RUNNER" -targetargs:"$CS_TESTS" ` + -target:"$CS_RUNNER" -targetargs:"$CS_TESTS --labels=All" ` -filter:"+[*]Python.Runtime*" ` -returntargetcode $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests failed" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Embedded tests completed" + + # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 + # the test is only built using modern stack + if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { + # Run C# Performance tests + Write-Host ("Starting performance tests") -ForegroundColor "Green" + if ($XPLAT) { + $CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll" + } + else { + $CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll" + } + &"$CS_RUNNER" "$CS_PERF_TESTS" + $CS_PERF_STATUS = $LastExitCode + if ($CS_PERF_STATUS -ne 0) { + Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime "Performance tests (C#) completed" + } + } else { + Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" + Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow" + $CS_PERF_STATUS = 0 + } } -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ if ($env:PLATFORM -eq "x64") { $DOTNET_CMD = "dotnet" } @@ -54,15 +101,20 @@ if ($env:BUILD_OPTS -eq "--xplat"){ # Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core) Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green" - &$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll + &$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll" $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime ".NET Core 2.0 tests completed" } } +Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green" + # Set exit code to fail if either Python or Embedded tests failed -if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) { +if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { Write-Host "Tests failed" -ForegroundColor "Red" $host.SetShouldExit(1) } diff --git a/conda.recipe/README.md b/conda.recipe/README.md deleted file mode 100644 index 42241999f..000000000 --- a/conda.recipe/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Conda Recipe - -The files here are needed to build Python.Net with conda - -http://conda.pydata.org/docs/building/recipe.html diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat deleted file mode 100644 index 27b55f2de..000000000 --- a/conda.recipe/bld.bat +++ /dev/null @@ -1,6 +0,0 @@ -:: build it - -:: set path to modern MSBuild -set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\MSBuild\15.0\Bin;%PATH% - -%PYTHON% setup.py install diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index 3641185bb..000000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -package: - name: pythonnet - version: "2.4.0.dev0" - -build: - skip: True # [not win] - number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} - {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 - {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} - -source: - git_url: ../ - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - imports: - - clr - -about: - home: https://github.com/pythonnet/pythonnet - license: MIT - license_file: LICENSE diff --git a/demo/wordpad.py b/demo/wordpad.py index 876372100..409c8ad4d 100644 --- a/demo/wordpad.py +++ b/demo/wordpad.py @@ -382,7 +382,7 @@ def InitializeComponent(self): self.label1.Size = System.Drawing.Size(296, 140) self.label1.TabIndex = 2 self.label1.Text = "Python Wordpad - an example winforms " \ - "application using Python for .NET" + "application using Python.NET" self.AutoScaleBaseSize = System.Drawing.Size(5, 13) self.ClientSize = System.Drawing.Size(300, 150) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index f2015e480..0fcd6923d 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -1,189 +1,401 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.3 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29102.190 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src/runtime/Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src\runtime\Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src/embed_tests/Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src\embed_tests\Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src/clrmodule/clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src\clrmodule\clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src/console/Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src\console\Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src/testing/Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src\testing\Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + .gitignore = .gitignore + CHANGELOG.md = CHANGELOG.md + README.rst = README.rst + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" + ProjectSection(SolutionItems) = preProject + .travis.yml = .travis.yml + appveyor.yml = appveyor.yml + ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 + ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" + ProjectSection(SolutionItems) = preProject + setup.py = setup.py + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}" + ProjectSection(SolutionItems) = preProject + conda.recipe\bld.bat = conda.recipe\bld.bat + conda.recipe\meta.yaml = conda.recipe\meta.yaml + conda.recipe\README.md = conda.recipe\README.md + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + DebugMono|Any CPU = DebugMono|Any CPU DebugMono|x64 = DebugMono|x64 DebugMono|x86 = DebugMono|x86 + DebugMonoPY3|Any CPU = DebugMonoPY3|Any CPU DebugMonoPY3|x64 = DebugMonoPY3|x64 DebugMonoPY3|x86 = DebugMonoPY3|x86 + DebugWin|Any CPU = DebugWin|Any CPU DebugWin|x64 = DebugWin|x64 DebugWin|x86 = DebugWin|x86 + DebugWinPY3|Any CPU = DebugWinPY3|Any CPU DebugWinPY3|x64 = DebugWinPY3|x64 DebugWinPY3|x86 = DebugWinPY3|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + ReleaseMono|Any CPU = ReleaseMono|Any CPU ReleaseMono|x64 = ReleaseMono|x64 ReleaseMono|x86 = ReleaseMono|x86 + ReleaseMonoPY3|Any CPU = ReleaseMonoPY3|Any CPU ReleaseMonoPY3|x64 = ReleaseMonoPY3|x64 ReleaseMonoPY3|x86 = ReleaseMonoPY3|x86 + ReleaseWin|Any CPU = ReleaseWin|Any CPU ReleaseWin|x64 = ReleaseWin|x64 ReleaseWin|x86 = ReleaseWin|x86 + ReleaseWinPY3|Any CPU = ReleaseWinPY3|Any CPU ReleaseWinPY3|x64 = ReleaseWinPY3|x64 ReleaseWinPY3|x86 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.Build.0 = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.Build.0 = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.Build.0 = DebugMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.Build.0 = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.Build.0 = DebugWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.Build.0 = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x64.ActiveCfg = DebugMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.ActiveCfg = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.Build.0 = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.Build.0 = DebugWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.Build.0 = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.ActiveCfg = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.Build.0 = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.Build.0 = DebugMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.ActiveCfg = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.Build.0 = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.Build.0 = DebugWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.Build.0 = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.Build.0 = DebugMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.Build.0 = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.Build.0 = DebugWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.Build.0 = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.Build.0 = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/pythonnet.sln b/pythonnet.sln index c5afd66c3..d920d4261 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25420.1 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30204.135 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.Runtime", "src\runtime\Python.Runtime.csproj", "{097B4AC0-74E9-4C58-BCF8-C69746EC8271}" EndProject @@ -32,38 +32,38 @@ Global ReleaseWinPY3|x86 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.Build.0 = DebugMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.Build.0 = DebugMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.Build.0 = DebugWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.Build.0 = DebugWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x64.Build.0 = DebugMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMono|x86.Build.0 = DebugMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x64.Build.0 = DebugWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x64.ActiveCfg = DebugMono|x64 {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x64.Build.0 = DebugMono|x64 {6F401A34-273B-450F-9A4C-13550BE0767B}.DebugMono|x86.ActiveCfg = DebugMono|x86 @@ -188,6 +188,9 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9E7459DF-55B1-4DF0-A0FB-251ABEEBD022} + EndGlobalSection GlobalSection(MonoDevelopProperties) = preSolution StartupItem = src\console\Console.csproj Policies = $0 diff --git a/setup.cfg b/setup.cfg index 38aa3eb3d..19c6f9fc9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -# [bumpversion] comments. bumpversion deleted all comments on its file. -# Don't combine `.bumpversion.cfg` with `setup.cfg`. Messes up formatting. -# Don't use `first_value = 1`. It will break `release` bump -# Keep `optional = dummy` needed to bump to release. -# See: https://github.com/peritus/bumpversion/issues/59 - [tool:pytest] xfail_strict = True # -r fsxX: show extra summary info for: (f)ailed, (s)kip, (x)failed, (X)passed diff --git a/setup.py b/setup.py index 1b6f07ea6..ca4f49a63 100644 --- a/setup.py +++ b/setup.py @@ -15,16 +15,20 @@ import sysconfig from distutils import spawn from distutils.command import install, build, build_ext, install_data, install_lib -from wheel import bdist_wheel from setuptools import Extension, setup +try: + from wheel import bdist_wheel +except ImportError: + bdist_wheel = None + # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug VERBOSITY = "normal" # quiet, minimal, normal, detailed, diagnostic -is_64bits = sys.maxsize > 2**32 +is_64bits = sys.maxsize > 2 ** 32 DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" ARCH = "x64" if is_64bits else "x86" PY_MAJOR = sys.version_info[0] @@ -32,63 +36,113 @@ ############################################################################### # Windows Keys Constants for MSBUILD tools -RegKey = collections.namedtuple('RegKey', 'sdk_name key value_name suffix') +RegKey = collections.namedtuple("RegKey", "sdk_name key value_name suffix") vs_python = "Programs\\Common\\Microsoft\\Visual C++ for Python\\9.0\\WinSDK" vs_root = "SOFTWARE\\Microsoft\\MSBuild\\ToolsVersions\\{0}" sdks_root = "SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v{0}Win32Tools" kits_root = "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots" kits_suffix = os.path.join("bin", ARCH) -WIN_SDK_KEYS = ( - RegKey(sdk_name="Windows Kit 10.0", key=kits_root, - value_name="KitsRoot10", suffix=os.path.join("bin", "10.0.16299.0", ARCH)), - - RegKey(sdk_name="Windows Kit 10.0", key=kits_root, - value_name="KitsRoot10", suffix=os.path.join("bin", "10.0.15063.0", ARCH)), - - RegKey(sdk_name="Windows Kit 10.0", key=kits_root, - value_name="KitsRoot10", suffix=kits_suffix), - - RegKey(sdk_name="Windows Kit 8.1", key=kits_root, - value_name="KitsRoot81", suffix=kits_suffix), - - RegKey(sdk_name="Windows Kit 8.0", key=kits_root, - value_name="KitsRoot", suffix=kits_suffix), - - RegKey(sdk_name="Windows SDK 7.1A", key=sdks_root.format("7.1A\\WinSDK-"), - value_name="InstallationFolder", suffix=""), - - RegKey(sdk_name="Windows SDK 7.1", key=sdks_root.format("7.1\\WinSDK"), - value_name="InstallationFolder", suffix=""), - - RegKey(sdk_name="Windows SDK 7.0A", key=sdks_root.format("7.0A\\WinSDK-"), - value_name="InstallationFolder", suffix=""), - - RegKey(sdk_name="Windows SDK 7.0", key=sdks_root.format("7.0\\WinSDK"), - value_name="InstallationFolder", suffix=""), - - RegKey(sdk_name="Windows SDK 6.0A", key=sdks_root.format("6.0A\\WinSDK"), - value_name="InstallationFolder", suffix=""), -) +WIN_SDK_KEYS = [ + RegKey( + sdk_name="Windows Kit 10.0", + key=kits_root, + value_name="KitsRoot10", + suffix=os.path.join("bin", "10.0.16299.0", ARCH), + ), + RegKey( + sdk_name="Windows Kit 10.0", + key=kits_root, + value_name="KitsRoot10", + suffix=os.path.join("bin", "10.0.15063.0", ARCH), + ), + RegKey( + sdk_name="Windows Kit 10.0", + key=kits_root, + value_name="KitsRoot10", + suffix=kits_suffix, + ), + RegKey( + sdk_name="Windows Kit 8.1", + key=kits_root, + value_name="KitsRoot81", + suffix=kits_suffix, + ), + RegKey( + sdk_name="Windows Kit 8.0", + key=kits_root, + value_name="KitsRoot", + suffix=kits_suffix, + ), + RegKey( + sdk_name="Windows SDK 7.1A", + key=sdks_root.format("7.1A\\WinSDK-"), + value_name="InstallationFolder", + suffix="", + ), + RegKey( + sdk_name="Windows SDK 7.1", + key=sdks_root.format("7.1\\WinSDK"), + value_name="InstallationFolder", + suffix="", + ), + RegKey( + sdk_name="Windows SDK 7.0A", + key=sdks_root.format("7.0A\\WinSDK-"), + value_name="InstallationFolder", + suffix="", + ), + RegKey( + sdk_name="Windows SDK 7.0", + key=sdks_root.format("7.0\\WinSDK"), + value_name="InstallationFolder", + suffix="", + ), + RegKey( + sdk_name="Windows SDK 6.0A", + key=sdks_root.format("6.0A\\WinSDK"), + value_name="InstallationFolder", + suffix="", + ), +] VS_KEYS = ( - RegKey(sdk_name="MSBuild 15", key=vs_root.format("15.0"), - value_name="MSBuildToolsPath", suffix=""), - - RegKey(sdk_name="MSBuild 14", key=vs_root.format("14.0"), - value_name="MSBuildToolsPath", suffix=""), - - RegKey(sdk_name="MSBuild 12", key=vs_root.format("12.0"), - value_name="MSBuildToolsPath", suffix=""), - - RegKey(sdk_name="MSBuild 4", key=vs_root.format("4.0"), - value_name="MSBuildToolsPath", suffix=""), - - RegKey(sdk_name="MSBuild 3.5", key=vs_root.format("3.5"), - value_name="MSBuildToolsPath", suffix=""), - - RegKey(sdk_name="MSBuild 2.0", key=vs_root.format("2.0"), - value_name="MSBuildToolsPath", suffix=""), + RegKey( + sdk_name="MSBuild 15", + key=vs_root.format("15.0"), + value_name="MSBuildToolsPath", + suffix="", + ), + RegKey( + sdk_name="MSBuild 14", + key=vs_root.format("14.0"), + value_name="MSBuildToolsPath", + suffix="", + ), + RegKey( + sdk_name="MSBuild 12", + key=vs_root.format("12.0"), + value_name="MSBuildToolsPath", + suffix="", + ), + RegKey( + sdk_name="MSBuild 4", + key=vs_root.format("4.0"), + value_name="MSBuildToolsPath", + suffix="", + ), + RegKey( + sdk_name="MSBuild 3.5", + key=vs_root.format("3.5"), + value_name="MSBuildToolsPath", + suffix="", + ), + RegKey( + sdk_name="MSBuild 2.0", + key=vs_root.format("2.0"), + value_name="MSBuildToolsPath", + suffix="", + ), ) @@ -96,8 +150,6 @@ def _check_output(*args, **kwargs): """Check output wrapper for py2/py3 compatibility""" output = subprocess.check_output(*args, **kwargs) - if PY_MAJOR == 2: - return output return output.decode("ascii") @@ -108,19 +160,19 @@ def _get_interop_filename(): required to generate the file. """ interop_filename = "interop{0}{1}{2}.cs".format( - PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "")) + PY_MAJOR, PY_MINOR, getattr(sys, "abiflags", "") + ) return os.path.join("src", "runtime", interop_filename) def _get_source_files(): """Walk project and collect the files needed for ext_module""" - for ext in (".sln", ): + for ext in (".sln",): for path in glob.glob("*" + ext): yield path for root, dirnames, filenames in os.walk("src"): - for ext in (".cs", ".csproj", ".snk", ".config", - ".py", ".c", ".h", ".ico"): + for ext in (".cs", ".csproj", ".snk", ".config", ".py", ".c", ".h", ".ico"): for filename in fnmatch.filter(filenames, "*" + ext): yield os.path.join(root, filename) @@ -132,11 +184,8 @@ def _get_source_files(): def _get_long_description(): """Helper to populate long_description for pypi releases""" - try: - import pypandoc - return pypandoc.convert('README.md', 'rst') - except ImportError: - return '.Net and Mono integration for Python' + return open("README.rst").read() + def _update_xlat_devtools(): global DEVTOOLS @@ -145,10 +194,40 @@ def _update_xlat_devtools(): elif DEVTOOLS == "Mono": DEVTOOLS = "dotnet" + +def _collect_installed_windows_kits_v10(winreg): + """Adds the installed Windows 10 kits to WIN_SDK_KEYS """ + global WIN_SDK_KEYS + installed_kits = [] + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, kits_root, 0, winreg.KEY_READ + ) as key: + i = 0 + while True: + try: + installed_kits.append(winreg.EnumKey(key, i)) + i += 1 + except WindowsError: + break + + def make_reg_key(version): + return RegKey( + sdk_name="Windows Kit 10.0", + key=kits_root, + value_name="KitsRoot10", + suffix=os.path.join("bin", version, ARCH), + ) + + WIN_SDK_KEYS += [make_reg_key(e) for e in installed_kits if e.startswith("10.")] + + # Make sure this function won't be called again + _collect_installed_windows_kits_v10 = lambda: None + + class BuildExtPythonnet(build_ext.build_ext): - user_options = build_ext.build_ext.user_options + [ - ('xplat', None, None) - ] + user_options = build_ext.build_ext.user_options + [("xplat", None, None)] + def initialize_options(self): build_ext.build_ext.initialize_options(self) self.xplat = None @@ -158,7 +237,7 @@ def finalize_options(self): def build_extension(self, ext): if self.xplat: - _update_xlat_devtools() + _update_xlat_devtools() """Builds the .pyd file using msbuild or xbuild""" if ext.name != "clr": @@ -174,16 +253,11 @@ def build_extension(self, ext): # Up to Python 3.2 sys.maxunicode is used to determine the size of # Py_UNICODE, but from 3.3 onwards Py_UNICODE is a typedef of wchar_t. - # TODO: Is this doing the right check for Py27? - if sys.version_info[:2] <= (3, 2): - unicode_width = 2 if sys.maxunicode < 0x10FFFF else 4 - else: - import ctypes - unicode_width = ctypes.sizeof(ctypes.c_wchar) + import ctypes + unicode_width = ctypes.sizeof(ctypes.c_wchar) defines = [ "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR), - "PYTHON{0}".format(PY_MAJOR), # Python Major Version "UCS{0}".format(unicode_width), ] @@ -192,7 +266,6 @@ def build_extension(self, ext): if sys.platform != "win32" and (DEVTOOLS == "Mono" or DEVTOOLS == "dotnet"): on_darwin = sys.platform == "darwin" - defines.append("MONO_OSX" if on_darwin else "MONO_LINUX") # Check if --enable-shared was set when Python was built enable_shared = sysconfig.get_config_var("Py_ENABLE_SHARED") @@ -200,12 +273,15 @@ def build_extension(self, ext): # Double-check if libpython is linked dynamically with python ldd_cmd = ["otool", "-L"] if on_darwin else ["ldd"] lddout = _check_output(ldd_cmd + [sys.executable]) - if 'libpython' not in lddout: + if "libpython" not in lddout: enable_shared = False if not enable_shared: defines.append("PYTHON_WITHOUT_ENABLE_SHARED") + if sys.platform == "win32": + defines.append("WINDOWS") + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.append("PYTHON_WITH_PYDEBUG") @@ -222,36 +298,42 @@ def build_extension(self, ext): if DEVTOOLS == "MsDev": _xbuild = '"{0}"'.format(self._find_msbuild_tool("msbuild.exe")) _config = "{0}Win".format(CONFIG) - _solution_file = 'pythonnet.sln' + _solution_file = "pythonnet.sln" _custom_define_constants = False elif DEVTOOLS == "MsDev15": - _xbuild = '"{0}"'.format(self._find_msbuild_tool_15()) + _xbuild = '"{0}"'.format(self._find_msbuild_tool_15()) _config = "{0}Win".format(CONFIG) - _solution_file = 'pythonnet.15.sln' + _solution_file = "pythonnet.15.sln" _custom_define_constants = True elif DEVTOOLS == "Mono": - _xbuild = 'xbuild' + _xbuild = "xbuild" _config = "{0}Mono".format(CONFIG) - _solution_file = 'pythonnet.sln' + _solution_file = "pythonnet.sln" _custom_define_constants = False elif DEVTOOLS == "dotnet": - _xbuild = 'dotnet msbuild' + _xbuild = "dotnet msbuild" _config = "{0}Mono".format(CONFIG) - _solution_file = 'pythonnet.15.sln' + _solution_file = "pythonnet.15.sln" _custom_define_constants = True else: raise NotImplementedError( - "DevTool {0} not supported (use MsDev/MsDev15/Mono/dotnet)".format(DEVTOOLS)) + "DevTool {0} not supported (use MsDev/MsDev15/Mono/dotnet)".format( + DEVTOOLS + ) + ) cmd = [ _xbuild, _solution_file, - '/p:Configuration={}'.format(_config), - '/p:Platform={}'.format(ARCH), - '/p:{}DefineConstants="{}"'.format('Custom' if _custom_define_constants else '','%3B'.join(defines)), + "/p:Configuration={}".format(_config), + "/p:Platform={}".format(ARCH), + '/p:{}DefineConstants="{}"'.format( + "Custom" if _custom_define_constants else "", "%3B".join(defines) + ), '/p:PythonBuildDir="{}"'.format(os.path.abspath(dest_dir)), '/p:PythonInteropFile="{}"'.format(os.path.basename(interop_file)), - '/verbosity:{}'.format(VERBOSITY), + "/p:PackageId=pythonnet_py{0}{1}_{2}".format(PY_MAJOR, PY_MINOR, ARCH), + "/verbosity:{}".format(VERBOSITY), ] manifest = self._get_manifest(dest_dir) @@ -264,7 +346,26 @@ def build_extension(self, ext): subprocess.check_call(" ".join(cmd + ["/t:Clean"]), shell=use_shell) subprocess.check_call(" ".join(cmd + ["/t:Build"]), shell=use_shell) if DEVTOOLS == "MsDev15" or DEVTOOLS == "dotnet": - subprocess.check_call(" ".join(cmd + ['"/t:Console_15:publish;Python_EmbeddingTest_15:publish"', "/p:TargetFramework=netcoreapp2.0"]), shell=use_shell) + subprocess.check_call( + " ".join( + cmd + + [ + '"/t:Console_15:publish;Python_EmbeddingTest_15:publish"', + "/p:TargetFramework=netcoreapp2.0", + ] + ), + shell=use_shell, + ) + subprocess.check_call( + " ".join( + cmd + + [ + '"/t:Python_PerformanceTests:publish"', + "/p:TargetFramework=net461", + ] + ), + shell=use_shell, + ) if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet": self._build_monoclr() @@ -273,8 +374,11 @@ def _get_manifest(self, build_dir): return mt = self._find_msbuild_tool("mt.exe", use_windows_sdk=True) manifest = os.path.abspath(os.path.join(build_dir, "app.manifest")) - cmd = [mt, '-inputresource:"{0}"'.format(sys.executable), - '-out:"{0}"'.format(manifest)] + cmd = [ + mt, + '-inputresource:"{0}"'.format(sys.executable), + '-out:"{0}"'.format(manifest), + ] self.debug_print("Extracting manifest from {}".format(sys.executable)) subprocess.check_call(" ".join(cmd), shell=False) return manifest @@ -288,20 +392,16 @@ def _build_monoclr(self): return raise mono_cflags = _check_output("pkg-config --cflags mono-2", shell=True) - glib_libs = _check_output("pkg-config --libs glib-2.0", shell=True) - glib_cflags = _check_output("pkg-config --cflags glib-2.0", shell=True) - cflags = mono_cflags.strip() + " " + glib_cflags.strip() - libs = mono_libs.strip() + " " + glib_libs.strip() + cflags = mono_cflags.strip() + libs = mono_libs.strip() # build the clr python module clr_ext = Extension( "clr", - sources=[ - "src/monoclr/pynetinit.c", - "src/monoclr/clrmod.c" - ], + language="c++", + sources=["src/monoclr/pynetinit.c", "src/monoclr/clrmod.c"], extra_compile_args=cflags.split(" "), - extra_link_args=libs.split(" ") + extra_link_args=libs.split(" "), ) build_ext.build_ext.build_extension(self, clr_ext) @@ -316,7 +416,9 @@ def _install_packages(self): elif DEVTOOLS == "dotnet": _config = "{0}Mono".format(CONFIG) - cmd = "dotnet msbuild /t:Restore pythonnet.15.sln /p:Configuration={0} /p:Platform={1}".format(_config, ARCH) + cmd = "dotnet msbuild /t:Restore pythonnet.15.sln /p:Configuration={0} /p:Platform={1}".format( + _config, ARCH + ) self.debug_print("Updating packages with xplat: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) else: @@ -331,7 +433,9 @@ def _install_packages(self): try: # msbuild=14 is mainly for Mono issues - cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) + cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format( + nuget + ) self.debug_print("Installing packages: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) except: @@ -344,17 +448,24 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the Microsoft build tools""" # trying to search path with help of vswhere when MSBuild 15.0 and higher installed. - if tool=="msbuild.exe" and use_windows_sdk==False: + if tool == "msbuild.exe" and use_windows_sdk == False: try: - basePathes = subprocess.check_output( - ["tools\\vswhere\\vswhere.exe", "-latest", - "-version", "[15.0, 16.0)", - "-requires", "Microsoft.Component.MSBuild", - "-property", "InstallationPath"]).splitlines() - if len(basePathes): - return os.path.join(basePathes[0].decode(sys.stdout.encoding or "utf-8"), "MSBuild", "15.0", "Bin", "MSBuild.exe") + basePaths = subprocess.check_output( + [ + "tools\\vswhere\\vswhere.exe", + "-latest", + "-version", + "[15.0,)", + "-requires", + "Microsoft.Component.MSBuild", + "-find", + "MSBuild\**\Bin\MSBuild.exe", + ] + ).splitlines() + if len(basePaths): + return basePaths[0].decode(sys.stdout.encoding or "utf-8") except: - pass # keep trying to search by old method. + pass # keep trying to search by old method. # Search in PATH first path = spawn.find_executable(tool) @@ -362,10 +473,9 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): return path # Search within registry to find build tools - try: # PY2 - import _winreg as winreg - except ImportError: # PY3 - import winreg + import winreg + + _collect_installed_windows_kits_v10(winreg) keys_to_check = WIN_SDK_KEYS if use_windows_sdk else VS_KEYS hklm = winreg.HKEY_LOCAL_MACHINE @@ -377,8 +487,9 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): continue path = os.path.join(val, rkey.suffix, tool) if os.path.exists(path): - self.debug_print("Using {0} from {1}".format( - tool, rkey.sdk_name)) + self.debug_print( + "Using {0} from {1}".format(tool, rkey.sdk_name) + ) return path except WindowsError: # Key doesn't exist @@ -401,23 +512,35 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): def _find_msbuild_tool_15(self): """Return full path to one of the Microsoft build tools""" try: - basePathes = subprocess.check_output( - ["tools\\vswhere\\vswhere.exe", "-latest", - "-version", "[15.0, 16.0)", - "-requires", "Microsoft.Component.MSBuild", - "-property", "InstallationPath"]).splitlines() - if len(basePathes): - return os.path.join(basePathes[0].decode(sys.stdout.encoding or "utf-8"), "MSBuild", "15.0", "Bin", "MSBuild.exe") + basePaths = subprocess.check_output( + [ + "tools\\vswhere\\vswhere.exe", + "-latest", + "-version", + "[15.0,)", + "-requires", + "Microsoft.Component.MSBuild", + "-find", + "MSBuild\**\Bin\MSBuild.exe", + ] + ).splitlines() + if len(basePaths): + return basePaths[0].decode(sys.stdout.encoding or "utf-8") else: raise RuntimeError("MSBuild >=15.0 could not be found.") except subprocess.CalledProcessError as e: - raise RuntimeError("MSBuild >=15.0 could not be found. {0}".format(e.output)) + raise RuntimeError( + "MSBuild >=15.0 could not be found. {0}".format(e.output) + ) + class InstallLibPythonnet(install_lib.install_lib): def install(self): if not os.path.isdir(self.build_dir): - self.warn("'{0}' does not exist -- no Python modules" - " to install".format(self.build_dir)) + self.warn( + "'{0}' does not exist -- no Python modules" + " to install".format(self.build_dir) + ) return if not os.path.exists(self.install_dir): @@ -425,8 +548,7 @@ def install(self): # only copy clr.pyd/.so for srcfile in glob.glob(os.path.join(self.build_dir, "clr.*")): - destfile = os.path.join( - self.install_dir, os.path.basename(srcfile)) + destfile = os.path.join(self.install_dir, os.path.basename(srcfile)) self.copy_file(srcfile, destfile) @@ -435,8 +557,7 @@ def run(self): build_cmd = self.get_finalized_command("build_ext") install_cmd = self.get_finalized_command("install") build_lib = os.path.abspath(build_cmd.build_lib) - install_platlib = os.path.relpath( - install_cmd.install_platlib, self.install_dir) + install_platlib = os.path.relpath(install_cmd.install_platlib, self.install_dir) for i, data_files in enumerate(self.data_files): if isinstance(data_files, str): @@ -449,10 +570,10 @@ def run(self): return install_data.install_data.run(self) + class InstallPythonnet(install.install): - user_options = install.install.user_options + [ - ('xplat', None, None) - ] + user_options = install.install.user_options + [("xplat", None, None)] + def initialize_options(self): install.install.initialize_options(self) self.xplat = None @@ -465,71 +586,64 @@ def run(self): _update_xlat_devtools() return install.install.run(self) -class BDistWheelPythonnet(bdist_wheel.bdist_wheel): - user_options = bdist_wheel.bdist_wheel.user_options + [ - ('xplat', None, None) - ] - def initialize_options(self): - bdist_wheel.bdist_wheel.initialize_options(self) - self.xplat = None +if bdist_wheel: + class BDistWheelPythonnet(bdist_wheel.bdist_wheel): + user_options = bdist_wheel.bdist_wheel.user_options + [("xplat", None, None)] - def finalize_options(self): - bdist_wheel.bdist_wheel.finalize_options(self) + def initialize_options(self): + bdist_wheel.bdist_wheel.initialize_options(self) + self.xplat = None - def run(self): - if self.xplat: - _update_xlat_devtools() - return bdist_wheel.bdist_wheel.run(self) + def finalize_options(self): + bdist_wheel.bdist_wheel.finalize_options(self) + + def run(self): + if self.xplat: + _update_xlat_devtools() + return bdist_wheel.bdist_wheel.run(self) ############################################################################### + + setupdir = os.path.dirname(__file__) if setupdir: os.chdir(setupdir) -setup_requires = [] -if not os.path.exists(_get_interop_filename()): - setup_requires.append("pycparser") +cmdclass={ + "install": InstallPythonnet, + "build_ext": BuildExtPythonnet, + "install_lib": InstallLibPythonnet, + "install_data": InstallDataPythonnet, +} +if bdist_wheel: + cmdclass["bdist_wheel"] = BDistWheelPythonnet setup( name="pythonnet", - version="2.4.0.dev0", + version="3.0.0dev1", description=".Net and Mono integration for Python", - url='https://pythonnet.github.io/', - license='MIT', - author="The Python for .Net developers", - author_email="pythondotnet@python.org", - setup_requires=setup_requires, + url="https://pythonnet.github.io/", + license="MIT", + author="The Contributors of the Python.NET Project", + author_email="pythonnet@python.org", + install_requires=["pycparser"], long_description=_get_long_description(), - ext_modules=[ - Extension("clr", sources=list(_get_source_files())) - ], - data_files=[ - ("{install_platlib}", [ - "{build_lib}/Python.Runtime.dll", - ]), - ], - cmdclass={ - "install": InstallPythonnet, - "build_ext": BuildExtPythonnet, - "install_lib": InstallLibPythonnet, - "install_data": InstallDataPythonnet, - "bdist_wheel": BDistWheelPythonnet - }, + ext_modules=[Extension("clr", sources=list(_get_source_files()))], + data_files=[("{install_platlib}", ["{build_lib}/Python.Runtime.dll"])], + cmdclass=cmdclass, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: C#', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', - 'Operating System :: MacOS :: MacOS X', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: C#", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", ], zip_safe=False, ) diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..ab79c56eb 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -8,8 +8,8 @@ // associated with an assembly. [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("pythonnet")] -[assembly: AssemblyProduct("Python for .NET")] -[assembly: AssemblyCopyright("Copyright (c) 2006-2017 the contributors of the 'Python for .NET' project")] +[assembly: AssemblyProduct("Python.NET")] +[assembly: AssemblyCopyright("Copyright (c) 2006-2020 the contributors of the Python.NET project")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("3.0.0")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..e19e58594 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -30,13 +30,8 @@ public class clrModule { -#if PYTHON3 [DllExport("PyInit_clr", CallingConvention.StdCall)] public static IntPtr PyInit_clr() -#elif PYTHON2 - [DllExport("initclr", CallingConvention.StdCall)] - public static void initclr() -#endif { DebugPrint("Attempting to load 'Python.Runtime' using standard binding rules."); #if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN @@ -53,7 +48,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("2.5.0"), #endif CultureInfo = CultureInfo.InvariantCulture }; @@ -95,11 +90,7 @@ public static void initclr() catch (InvalidOperationException) { DebugPrint("Could not load 'Python.Runtime'."); -#if PYTHON3 return IntPtr.Zero; -#elif PYTHON2 - return; -#endif } } @@ -107,11 +98,7 @@ public static void initclr() // So now we get the PythonEngine and execute the InitExt method on it. Type pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); -#if PYTHON3 return (IntPtr)pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); -#elif PYTHON2 - pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); -#endif } /// diff --git a/src/clrmodule/clrmodule.15.csproj b/src/clrmodule/clrmodule.15.csproj index 2585ffdd2..7fc9c2dda 100644 --- a/src/clrmodule/clrmodule.15.csproj +++ b/src/clrmodule/clrmodule.15.csproj @@ -9,7 +9,7 @@ clrmodule clrmodule clrmodule - 2.4.0 + 2.5.0 false false false diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj index 6e5ff4966..374948d89 100644 --- a/src/clrmodule/clrmodule.csproj +++ b/src/clrmodule/clrmodule.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ clrmodule bin\clrmodule.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -92,4 +92,4 @@ - + \ No newline at end of file diff --git a/src/clrmodule/packages.config b/src/clrmodule/packages.config index 2a95dc54d..bfd9f475d 100644 --- a/src/clrmodule/packages.config +++ b/src/clrmodule/packages.config @@ -1,4 +1,4 @@ - + - - + + \ No newline at end of file diff --git a/src/console/Console.15.csproj b/src/console/Console.15.csproj index ec5008036..3c51caa31 100644 --- a/src/console/Console.15.csproj +++ b/src/console/Console.15.csproj @@ -8,7 +8,7 @@ nPython Python.Runtime nPython - 2.4.0 + 2.5.0 false false false diff --git a/src/console/Console.csproj b/src/console/Console.csproj index ea88b6356..eb54c9209 100644 --- a/src/console/Console.csproj +++ b/src/console/Console.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Runtime bin\nPython.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -98,4 +98,4 @@ - + \ No newline at end of file diff --git a/src/embed_tests/CodecGroups.cs b/src/embed_tests/CodecGroups.cs new file mode 100644 index 000000000..5dd40210f --- /dev/null +++ b/src/embed_tests/CodecGroups.cs @@ -0,0 +1,132 @@ +namespace Python.EmbeddingTest +{ + using System; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class CodecGroups + { + [Test] + public void GetEncodersByType() + { + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + encoder1, + encoder2, + }; + + var got = group.GetEncoders(typeof(Uri)).ToArray(); + CollectionAssert.AreEqual(new[]{encoder1, encoder2}, got); + } + + [Test] + public void CanEncode() + { + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + new ObjectToEncoderInstanceEncoder(), + }; + + Assert.IsTrue(group.CanEncode(typeof(Tuple))); + Assert.IsTrue(group.CanEncode(typeof(Uri))); + Assert.IsFalse(group.CanEncode(typeof(string))); + } + + [Test] + public void Encodes() + { + var encoder0 = new ObjectToEncoderInstanceEncoder>(); + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + encoder0, + encoder1, + encoder2, + }; + + var uri = group.TryEncode(new Uri("data:")); + var clrObject = (CLRObject)ManagedType.GetManagedObject(uri.Handle); + Assert.AreSame(encoder1, clrObject.inst); + Assert.AreNotSame(encoder2, clrObject.inst); + + var tuple = group.TryEncode(Tuple.Create(1)); + clrObject = (CLRObject)ManagedType.GetManagedObject(tuple.Handle); + Assert.AreSame(encoder0, clrObject.inst); + } + + [Test] + public void GetDecodersByTypes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + var decoder = group.GetDecoder(pyfloat, typeof(string)); + Assert.AreSame(decoder2, decoder); + decoder = group.GetDecoder(pystr, typeof(string)); + Assert.IsNull(decoder); + decoder = group.GetDecoder(pyint, typeof(long)); + Assert.AreSame(decoder1, decoder); + } + [Test] + public void CanDecode() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.CanDecode(pyint, typeof(long))); + Assert.IsFalse(group.CanDecode(pyint, typeof(int))); + Assert.IsTrue(group.CanDecode(pyfloat, typeof(string))); + Assert.IsFalse(group.CanDecode(pystr, typeof(string))); + } + + [Test] + public void Decodes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.TryDecode(new PyInt(10), out long longResult)); + Assert.AreEqual(42, longResult); + Assert.IsTrue(group.TryDecode(new PyFloat(10), out string strResult)); + Assert.AreSame("atad:", strResult); + + Assert.IsFalse(group.TryDecode(new PyInt(10), out int _)); + } + + [SetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..18fcd32d1 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,123 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..dd103cf5c 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -10,7 +10,7 @@ Python.EmbeddingTest Python.EmbeddingTest Python.EmbeddingTest - 2.4.0 + 2.5.0 false false false @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 6 + 7.3 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT @@ -77,13 +77,18 @@ - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + - + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 6aa48becc..e1e66f54b 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,5 +1,5 @@ - + Debug AnyCPU @@ -9,12 +9,12 @@ Python.EmbeddingTest bin\Python.EmbeddingTest.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ $(SolutionDir)\bin\ - 6 + 7.3 true prompt @@ -70,24 +70,36 @@ - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + ..\..\packages\NUnit.3.12.0\lib\net40\nunit.framework.dll + + + ..\..\packages\System.ValueTuple.4.5.0\lib\portable-net40+sl4+win8+wp8\System.ValueTuple.dll + + ..\..\packages\System.ValueTuple.4.5.0\lib\netstandard1.0\System.ValueTuple.dll + + + + + + + @@ -125,4 +137,10 @@ + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs new file mode 100644 index 000000000..1d29e85c7 --- /dev/null +++ b/src/embed_tests/References.cs @@ -0,0 +1,55 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class References + { + private Py.GILState _gs; + + [SetUp] + public void SetUp() + { + _gs = Py.GIL(); + } + + [TearDown] + public void Dispose() + { + _gs.Dispose(); + } + + [Test] + public void MoveToPyObject_SetsNull() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + Assert.IsFalse(reference.IsNull()); + + using (reference.MoveToPyObject()) + Assert.IsTrue(reference.IsNull()); + } + finally + { + reference.Dispose(); + } + } + + [Test] + public void CanBorrowFromNewReference() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference)); + } + finally + { + reference.Dispose(); + } + } + } +} diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs new file mode 100644 index 000000000..454c97578 --- /dev/null +++ b/src/embed_tests/TestCallbacks.cs @@ -0,0 +1,33 @@ +using System; + +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest { + using Runtime = Python.Runtime.Runtime; + + public class TestCallbacks { + [OneTimeSetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void TestNoOverloadException() { + int passed = 0; + var aFunctionThatCallsIntoPython = new Action(value => passed = value); + using (Py.GIL()) { + dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); + var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); + Assert.AreEqual("TypeError", error.PythonTypeName); + string expectedArgTypes = "()"; + StringAssert.EndsWith(expectedArgTypes, error.Message); + } + } + } +} diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..072038d7a 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; @@ -17,6 +19,30 @@ public void Dispose() PythonEngine.Shutdown(); } + [Test] + public void ConvertListRoundTrip() + { + var list = new List { typeof(decimal), typeof(int) }; + var py = list.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(List), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, list); + } + + [Test] + public void ConvertPyListToArray() + { + var array = new List { typeof(decimal), typeof(int) }; + var py = array.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(Type[]), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, array); + } + [Test] public void TestConvertSingleToManaged( [Values(float.PositiveInfinity, float.NegativeInfinity, float.MinValue, float.MaxValue, float.NaN, @@ -44,5 +70,26 @@ public void TestConvertDoubleToManaged( Assert.IsTrue(converted); Assert.IsTrue(((double) convertedValue).Equals(testValue)); } + + [Test] + public void RawListProxy() + { + var list = new List {"hello", "world"}; + var listProxy = PyObject.FromManagedObject(list); + var clrObject = (CLRObject)ManagedType.GetManagedObject(listProxy.Handle); + Assert.AreSame(list, clrObject.inst); + } + + [Test] + public void RawPyObjectProxy() + { + var pyObject = "hello world!".ToPython(); + var pyObjectProxy = PyObject.FromManagedObject(pyObject); + var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy.Handle); + Assert.AreSame(pyObject, clrObject.inst); + + var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); + Assert.AreEqual(pyObject.Handle, proxiedHandle); + } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..f82767af1 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,253 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Linq; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private int _oldThreshold; + + [SetUp] + public void SetUp() + { + _oldThreshold = Finalizer.Instance.Threshold; + PythonEngine.Initialize(); + Exceptions.Clear(); + } + + [TearDown] + public void TearDown() + { + Finalizer.Instance.Threshold = _oldThreshold; + PythonEngine.Shutdown(); + } + + private static void FullGCCollect() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + GC.WaitForPendingFinalizers(); + } + + [Test] + public void CollectBasicObject() + { + Assert.IsTrue(Finalizer.Instance.Enable); + + Finalizer.Instance.Threshold = 1; + bool called = false; + var objectCount = 0; + EventHandler handler = (s, e) => + { + objectCount = e.ObjectCount; + called = true; + }; + + Assert.IsFalse(called, "The event handler was called before it was installed"); + Finalizer.Instance.CollectOnce += handler; + + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + } + FullGCCollect(); + // The object has been resurrected + Warn.If( + shortWeak.IsAlive, + "The referenced object is alive although it should have been collected", + shortWeak + ); + Assert.IsTrue( + longWeak.IsAlive, + "The reference object is not alive although it should still be", + longWeak + ); + + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count, "There should still be garbage around"); + Warn.Unless( + garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)), + $"The {nameof(longWeak)} reference doesn't show up in the garbage list", + garbage + ); + } + try + { + Finalizer.Instance.Collect(); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } + Assert.IsTrue(called, "The event handler was not called during finalization"); + Assert.GreaterOrEqual(objectCount, 1); + } + + private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + PyLong obj = new PyLong(1024); + shortWeak = new WeakReference(obj); + longWeak = new WeakReference(obj, true); + obj = null; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.Collect(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.Collect(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + + class MyPyObject : PyObject + { + public MyPyObject(IntPtr op) : base(op) + { + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + GC.SuppressFinalize(this); + throw new Exception("MyPyObject"); + } + internal static void CreateMyPyObject(IntPtr op) + { + Runtime.Runtime.XIncref(op); + new MyPyObject(op); + } + } + + [Test] + public void ErrorHandling() + { + bool called = false; + var errorMessage = ""; + EventHandler handleFunc = (sender, args) => + { + called = true; + errorMessage = args.Error.Message; + }; + Finalizer.Instance.Threshold = 1; + Finalizer.Instance.ErrorHandler += handleFunc; + try + { + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + var obj = (PyLong)longWeak.Target; + IntPtr handle = obj.Handle; + shortWeak = null; + longWeak = null; + MyPyObject.CreateMyPyObject(handle); + obj.Dispose(); + obj = null; + } + FullGCCollect(); + Finalizer.Instance.Collect(); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.ErrorHandler -= handleFunc; + } + Assert.AreEqual(errorMessage, "MyPyObject"); + } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Pass("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResolver += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResolver -= handler; + } + } + + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + + } +} diff --git a/src/embed_tests/TestGILState.cs b/src/embed_tests/TestGILState.cs new file mode 100644 index 000000000..bf6f02dc6 --- /dev/null +++ b/src/embed_tests/TestGILState.cs @@ -0,0 +1,33 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class TestGILState + { + /// + /// Ensure, that calling multiple times is safe + /// + [Test] + public void CanDisposeMultipleTimes() + { + using (var gilState = Py.GIL()) + { + for(int i = 0; i < 50; i++) + gilState.Dispose(); + } + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/TestInstanceWrapping.cs b/src/embed_tests/TestInstanceWrapping.cs new file mode 100644 index 000000000..8be207c00 --- /dev/null +++ b/src/embed_tests/TestInstanceWrapping.cs @@ -0,0 +1,58 @@ +using System; +using System.Globalization; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInstanceWrapping + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + // regression test for https://github.com/pythonnet/pythonnet/issues/811 + [Test] + public void OverloadResolution_UnknownToObject() + { + var overloaded = new Overloaded(); + using (Py.GIL()) + { + var o = overloaded.ToPython(); + + dynamic callWithSelf = PythonEngine.Eval("lambda o: o.ObjOrClass(object())"); + callWithSelf(o); + Assert.AreEqual(Overloaded.Object, overloaded.Value); + } + } + + class Base {} + class Derived: Base { } + + class Overloaded: Derived + { + public int Value { get; set; } + public void IntOrStr(int arg) => this.Value = arg; + public void IntOrStr(string arg) => + this.Value = int.Parse(arg, NumberStyles.Integer, CultureInfo.InvariantCulture); + + public const int Object = 1; + public const int ConcreteClass = 2; + public void ObjOrClass(object _) => this.Value = Object; + public void ObjOrClass(Overloaded _) => this.Value = ConcreteClass; + + public const int Base = ConcreteClass + 1; + public const int Derived = Base + 1; + public void BaseOrDerived(Base _) => this.Value = Base; + public void BaseOrDerived(Derived _) => this.Value = Derived; + } + } +} diff --git a/src/embed_tests/TestMethodBinder.cs b/src/embed_tests/TestMethodBinder.cs new file mode 100644 index 000000000..ba4096776 --- /dev/null +++ b/src/embed_tests/TestMethodBinder.cs @@ -0,0 +1,151 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestMethodBinder + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void ImplicitConversionToString() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyString(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyString impl: implicit to string", data); + } + } + + [Test] + public void ImplicitConversionToClass() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyClass('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyClass impl", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_String() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("string impl: input string", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_Class() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("TestImplicitConversion impl", data); + } + } + + public class CSharpModel + { + public virtual string OnlyClass(TestImplicitConversion data) + { + return "OnlyClass impl"; + } + + public virtual string OnlyString(string data) + { + return "OnlyString impl: " + data; + } + + public virtual string InvokeModel(string data) + { + return "string impl: " + data; + } + + public virtual string InvokeModel(TestImplicitConversion data) + { + return "TestImplicitConversion impl"; + } + } + + public class TestImplicitConversion + { + public static implicit operator string(TestImplicitConversion symbol) + { + return "implicit to string"; + } + public static implicit operator TestImplicitConversion(string symbol) + { + return new TestImplicitConversion(); + } + } + } +} diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d4952d4a3 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -57,5 +57,11 @@ def add(self, x, y): Assert.IsTrue(memberNames.Contains(expectedName), "Could not find member '{0}'.", expectedName); } } + + [Test] + public void InvokeNull() { + var list = PythonEngine.Eval("list"); + Assert.Throws(() => list.Invoke(new PyObject[] {null})); + } } } diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 49c15a3a1..701e698ec 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using NUnit.Framework; using Python.Runtime; @@ -293,24 +294,27 @@ public void TestImportScopeByName() [Test] public void TestVariables() { - (ps.Variables() as dynamic)["ee"] = new PyInt(200); - var a0 = ps.Get("ee"); - Assert.AreEqual(200, a0); + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); - ps.Exec("locals()['ee'] = 210"); - var a1 = ps.Get("ee"); - Assert.AreEqual(210, a1); + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); - ps.Exec("globals()['ee'] = 220"); - var a2 = ps.Get("ee"); - Assert.AreEqual(220, a2); + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); - using (var item = ps.Variables()) - { - item["ee"] = new PyInt(230); + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); } - var a3 = ps.Get("ee"); - Assert.AreEqual(230, a3); } /// @@ -324,49 +328,58 @@ public void TestThread() //should be removed. dynamic _ps = ps; var ts = PythonEngine.BeginAllowThreads(); - using (Py.GIL()) - { - _ps.res = 0; - _ps.bb = 100; - _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast - ps.Exec( - "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" - ); - } - int th_cnt = 3; - for (int i =0; i< th_cnt; i++) + try { - System.Threading.Thread th = new System.Threading.Thread(()=> + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "import threading\n" + + "lock = threading.Lock()\n" + + "def update():\n" + + " global res, th_cnt\n" + + " with lock:\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 100; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) { using (Py.GIL()) { - //ps.GetVariable("update")(); //call the scope function dynamicly - _ps.update(); + cnt = ps.Get("th_cnt"); } - }); - th.Start(); - } - //equivalent to Thread.Join, make the main thread join the GIL competition - int cnt = 0; - while(cnt != th_cnt) - { + Thread.Yield(); + } using (Py.GIL()) { - cnt = ps.Get("th_cnt"); + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); } - System.Threading.Thread.Sleep(10); } - using (Py.GIL()) + finally { - var result = ps.Get("res"); - Assert.AreEqual(101* th_cnt, result); + PythonEngine.EndAllowThreads(ts); } - PythonEngine.EndAllowThreads(ts); } } } diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 243349b82..6d2d5d6cc 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -180,12 +180,6 @@ public void SetProgramName() [Test] public void SetPythonPath() { - if (Runtime.Runtime.pyversion == "2.7") - { - // Assert.Skip outputs as a warning (ie. pending to fix) - Assert.Pass(); - } - PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); @@ -196,25 +190,5 @@ public void SetPythonPath() Assert.AreEqual(path, PythonEngine.PythonPath); PythonEngine.Shutdown(); } - - [Test] - public void SetPythonPathExceptionOn27() - { - if (Runtime.Runtime.pyversion != "2.7") - { - Assert.Pass(); - } - - PythonEngine.Initialize(); - string path = PythonEngine.PythonPath; - PythonEngine.Shutdown(); - - var ex = Assert.Throws(() => PythonEngine.PythonPath = "foo"); - Assert.AreEqual("Set PythonPath not supported on Python 2", ex.Message); - - PythonEngine.Initialize(); - Assert.AreEqual(path, PythonEngine.PythonPath); - PythonEngine.Shutdown(); - } } } diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 57a8d54af..000c32ca3 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -54,5 +54,41 @@ public void TestPythonErrorTypeName() Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError")); } } + + [Test] + public void TestPythonExceptionFormat() + { + try + { + PythonEngine.Exec("raise ValueError('Error!')"); + Assert.Fail("Exception should have been raised"); + } + catch (PythonException ex) + { + Assert.That(ex.Format(), Does.Contain("Traceback").And.Contains("(most recent call last):").And.Contains("ValueError: Error!")); + } + } + + [Test] + public void TestPythonExceptionFormatNoError() + { + var ex = new PythonException(); + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + + [Test] + public void TestPythonExceptionFormatNoTraceback() + { + try + { + var module = PythonEngine.ImportModule("really____unknown___module"); + Assert.Fail("Unknown module should not be loaded"); + } + catch (PythonException ex) + { + // ImportError/ModuleNotFoundError do not have a traceback when not running in a script + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + } } } diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..4129d3df3 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; namespace Python.EmbeddingTest { @@ -26,11 +28,8 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); + Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); + Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); // Don't shut down the runtime: if the python engine was initialized // but not shut down by another test, we'd end up in a bad state. @@ -39,7 +38,7 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); @@ -109,9 +108,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() // Create an instance of threading.Lock, which is one of the very few types that does not have the // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. var threading = Runtime.Runtime.PyImport_ImportModule("threading"); + Exceptions.ErrorCheck(threading); var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + Exceptions.ErrorCheck(threadingDict); var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); + if (lockType == IntPtr.Zero) + throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); + var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0)); + Exceptions.ErrorCheck(lockInstance); Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs index a4ef86913..931c44236 100644 --- a/src/embed_tests/TestTypeManager.cs +++ b/src/embed_tests/TestTypeManager.cs @@ -54,12 +54,12 @@ public static void TestMemoryMapping() // We can't use compiler flags because we compile with MONO_LINUX // while running on the Microsoft .NET Core during continuous // integration tests. - if (System.Type.GetType ("Mono.Runtime") != null) + /* if (System.Type.GetType ("Mono.Runtime") != null) { // Mono throws NRE instead of AccessViolationException for some reason. Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } + } */ } } } diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..485b3fff8 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - - + + + diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index acb3433de..6b2408745 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -39,7 +39,7 @@ public void SetUp() IntPtr str = Runtime.Runtime.PyString_FromString(testPath); IntPtr path = Runtime.Runtime.PySys_GetObject("path"); - Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.PyList_Append(new BorrowedReference(path), str); } [TearDown] diff --git a/src/monoclr/pynetclr.h b/src/monoclr/pynetclr.h index c5e181156..1863b1d43 100644 --- a/src/monoclr/pynetclr.h +++ b/src/monoclr/pynetclr.h @@ -7,12 +7,15 @@ #include #include #include -#include #define MONO_VERSION "v4.0.30319.1" #define MONO_DOMAIN "Python.Runtime" #define PR_ASSEMBLY "Python.Runtime.dll" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { MonoDomain *domain; @@ -27,7 +30,11 @@ typedef struct PyNet_Args *PyNet_Init(int); void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(gpointer user_data); +void main_thread_handler(PyNet_Args *user_data); char *PyNet_ExceptionToString(MonoObject *); +#ifdef __cplusplus +} +#endif + #endif // PYNET_CLR_H diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c index 8b49ddae3..149d52296 100644 --- a/src/monoclr/pynetinit.c +++ b/src/monoclr/pynetinit.c @@ -19,6 +19,15 @@ PyNet_Args *PyNet_Init(int ext) pn_args->shutdown = NULL; pn_args->module = NULL; +#ifdef __linux__ + // Force preload libmono-2.0 as global. Without this, on some systems + // symbols from libmono are not found by libmononative (which implements + // some of the System.* namespaces). Since the only happened on Linux so + // far, we hardcode the library name, load the symbols into the global + // namespace and leak the handle. + dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); +#endif + if (ext == 0) { pn_args->init_name = "Python.Runtime:Initialize()"; @@ -91,9 +100,9 @@ MonoMethod *getMethodFromClass(MonoClass *cls, char *name) return method; } -void main_thread_handler(gpointer user_data) +void main_thread_handler(PyNet_Args *user_data) { - PyNet_Args *pn_args = (PyNet_Args *)user_data; + PyNet_Args *pn_args = user_data; MonoMethod *init; MonoImage *pr_image; MonoClass *pythonengine; @@ -122,7 +131,7 @@ void main_thread_handler(gpointer user_data) strcpy(new_ld_library_path, py_libdir); strcat(new_ld_library_path, ":"); strcat(new_ld_library_path, ld_library_path); - setenv("LD_LIBRARY_PATH", py_libdir, 1); + setenv("LD_LIBRARY_PATH", new_ld_library_path, 1); } } @@ -241,7 +250,7 @@ void main_thread_handler(gpointer user_data) // Get string from a Mono exception char *PyNet_ExceptionToString(MonoObject *e) { - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", FALSE); + MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); mono_method_desc_free(mdesc); diff --git a/src/perf_tests/BaselineComparisonBenchmarkBase.cs b/src/perf_tests/BaselineComparisonBenchmarkBase.cs new file mode 100644 index 000000000..2388e3982 --- /dev/null +++ b/src/perf_tests/BaselineComparisonBenchmarkBase.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using Python.Runtime; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonBenchmarkBase + { + public BaselineComparisonBenchmarkBase() + { + Console.WriteLine($"CWD: {Environment.CurrentDirectory}"); + Console.WriteLine($"Using Python.Runtime from {typeof(PythonEngine).Assembly.Location} {typeof(PythonEngine).Assembly.GetName()}"); + + try { + PythonEngine.Initialize(); + Console.WriteLine("Python Initialized"); + if (PythonEngine.BeginAllowThreads() == IntPtr.Zero) + throw new PythonException(); + Console.WriteLine("Threading enabled"); + } + catch (Exception e) { + Console.WriteLine(e); + throw; + } + } + + static BaselineComparisonBenchmarkBase() + { + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + { + throw new ArgumentException( + "Required environment variable is missing", + BaselineComparisonConfig.EnvironmentVariableName); + } + + Console.WriteLine("Preloading " + pythonRuntimeDll); + Assembly.LoadFrom(pythonRuntimeDll); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) { + if (assembly.FullName.StartsWith("Python.Runtime")) + Console.WriteLine(assembly.Location); + foreach(var dependency in assembly.GetReferencedAssemblies()) + if (dependency.FullName.Contains("Python.Runtime")) { + Console.WriteLine($"{assembly} -> {dependency}"); + } + } + + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + if (!args.Name.StartsWith("Python.Runtime")) + return null; + + var preloaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "Python.Runtime"); + if (preloaded != null) return preloaded; + + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + return null; + + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs new file mode 100644 index 000000000..649bb56fd --- /dev/null +++ b/src/perf_tests/BaselineComparisonConfig.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Horology; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonConfig : ManualConfig + { + public const string EnvironmentVariableName = "PythonRuntimeDLL"; + + public BaselineComparisonConfig() + { + this.Options |= ConfigOptions.DisableOptimizationsValidator; + + string deploymentRoot = BenchmarkTests.DeploymentRoot; + + var baseJob = Job.Default + .WithLaunchCount(1) + .WithWarmupCount(3) + .WithMaxIterationCount(100) + .WithIterationTime(TimeInterval.FromMilliseconds(100)); + this.Add(baseJob + .WithId("baseline") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll")) + .WithBaseline(true)); + this.Add(baseJob + .WithId("new") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "new", "Python.Runtime.dll"))); + } + + static BaselineComparisonConfig() { + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + Console.WriteLine(args.Name); + if (!args.Name.StartsWith("Python.Runtime")) + return null; + string pythonRuntimeDll = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + pythonRuntimeDll = Path.Combine(BenchmarkTests.DeploymentRoot, "baseline", "Python.Runtime.dll"); + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs new file mode 100644 index 000000000..6e0afca69 --- /dev/null +++ b/src/perf_tests/BenchmarkTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Reflection; + +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using NUnit.Framework; + +namespace Python.PerformanceTests +{ + public class BenchmarkTests + { + Summary summary; + + [OneTimeSetUp] + public void SetUp() + { + Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); + this.summary = BenchmarkRunner.Run(); + Assert.IsNotEmpty(this.summary.Reports); + Assert.IsTrue( + condition: this.summary.Reports.All(r => r.Success), + message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above."); + } + + [Test] + public void ReadInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + [Test] + public void WriteInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + static double GetOptimisticPerfRatio( + IReadOnlyList reports, + [CallerMemberName] string methodName = null) + { + reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray(); + if (reports.Count == 0) + throw new ArgumentException( + message: $"No reports found for {methodName}. " + + "You have to match test method name to benchmark method name or " + + "pass benchmark method name explicitly", + paramName: nameof(methodName)); + + var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics; + var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics; + + double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5; + + return newTimeOptimistic / baseline.Mean; + } + + public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + + public static void AssertPerformanceIsBetterOrSame( + double actual, double target, + double wiggleRoom = 1.1, [CallerMemberName] string testName = null) { + double threshold = target * wiggleRoom; + Assert.LessOrEqual(actual, threshold, + $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})" + + ": perf result is higher than the failure threshold."); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj new file mode 100644 index 000000000..9d94281ef --- /dev/null +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -0,0 +1,48 @@ + + + + net461 + DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 + bin\ + + false + + x64;x86 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers + + + compile + + + + + + + + + + + + + + + + + + diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs new file mode 100644 index 000000000..ef668a911 --- /dev/null +++ b/src/perf_tests/PythonCallingNetBenchmark.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using BenchmarkDotNet.Attributes; +using Python.Runtime; + +namespace Python.PerformanceTests +{ + [Config(typeof(BaselineComparisonConfig))] + public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase + { + [Benchmark] + public void ReadInt64Property() + { + using (Py.GIL()) + { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + s += a.{nameof(NetObject.LongProperty)} +", locals: locals.Handle); + } + } + + [Benchmark] + public void WriteInt64Property() { + using (Py.GIL()) { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + a.{nameof(NetObject.LongProperty)} += i +", locals: locals.Handle); + } + } + } + + class NetObject + { + public long LongProperty { get; set; } = 42; + } +} diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs new file mode 100644 index 000000000..a3bf29056 --- /dev/null +++ b/src/runtime/BorrowedReference.cs @@ -0,0 +1,25 @@ +namespace Python.Runtime +{ + using System; + /// + /// Represents a reference to a Python object, that is being lent, and + /// can only be safely used until execution returns to the caller. + /// + readonly ref struct BorrowedReference + { + readonly IntPtr pointer; + public bool IsNull => this.pointer == IntPtr.Zero; + + /// Gets a raw pointer to the Python object + public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.pointer; + + /// + /// Creates new instance of from raw pointer. Unsafe. + /// + public BorrowedReference(IntPtr pointer) + { + this.pointer = pointer; + } + } +} diff --git a/src/runtime/Codecs/DecoderGroup.cs b/src/runtime/Codecs/DecoderGroup.cs new file mode 100644 index 000000000..8a290d5d4 --- /dev/null +++ b/src/runtime/Codecs/DecoderGroup.cs @@ -0,0 +1,78 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + [Obsolete(Util.UnstableApiMessage)] + public sealed class DecoderGroup: IPyObjectDecoder, IEnumerable + { + readonly List decoders = new List(); + + /// + /// Add specified decoder to the group + /// + public void Add(IPyObjectDecoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + + this.decoders.Add(item); + } + /// + /// Remove all decoders from the group + /// + public void Clear() => this.decoders.Clear(); + + /// + public bool CanDecode(PyObject objectType, Type targetType) + => this.decoders.Any(decoder => decoder.CanDecode(objectType, targetType)); + /// + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj is null) throw new ArgumentNullException(nameof(pyObj)); + + var decoder = this.GetDecoder(pyObj.GetPythonType(), typeof(T)); + if (decoder is null) + { + value = default; + return false; + } + return decoder.TryDecode(pyObj, out value); + } + + /// + public IEnumerator GetEnumerator() => this.decoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.decoders.GetEnumerator(); + } + + [Obsolete(Util.UnstableApiMessage)] + public static class DecoderGroupExtensions + { + /// + /// Gets a concrete instance of + /// (potentially selecting one from a collection), + /// that can decode from to , + /// or null if a matching decoder can not be found. + /// + [Obsolete(Util.UnstableApiMessage)] + public static IPyObjectDecoder GetDecoder( + this IPyObjectDecoder decoder, + PyObject objectType, Type targetType) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + return composite + .Select(nestedDecoder => nestedDecoder.GetDecoder(objectType, targetType)) + .FirstOrDefault(d => d != null); + } + + return decoder.CanDecode(objectType, targetType) ? decoder : null; + } + } +} diff --git a/src/runtime/Codecs/EncoderGroup.cs b/src/runtime/Codecs/EncoderGroup.cs new file mode 100644 index 000000000..a5708c0bb --- /dev/null +++ b/src/runtime/Codecs/EncoderGroup.cs @@ -0,0 +1,79 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + [Obsolete(Util.UnstableApiMessage)] + public sealed class EncoderGroup: IPyObjectEncoder, IEnumerable + { + readonly List encoders = new List(); + + /// + /// Add specified encoder to the group + /// + public void Add(IPyObjectEncoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + this.encoders.Add(item); + } + /// + /// Remove all encoders from the group + /// + public void Clear() => this.encoders.Clear(); + + /// + public bool CanEncode(Type type) => this.encoders.Any(encoder => encoder.CanEncode(type)); + /// + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + foreach (var encoder in this.GetEncoders(value.GetType())) + { + var result = encoder.TryEncode(value); + if (result != null) + { + return result; + } + } + + return null; + } + + /// + public IEnumerator GetEnumerator() => this.encoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.encoders.GetEnumerator(); + } + + [Obsolete(Util.UnstableApiMessage)] + public static class EncoderGroupExtensions + { + /// + /// Gets specific instances of + /// (potentially selecting one from a collection), + /// that can encode the specified . + /// + [Obsolete(Util.UnstableApiMessage)] + public static IEnumerable GetEncoders(this IPyObjectEncoder decoder, Type type) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + foreach (var nestedEncoder in composite) + foreach (var match in nestedEncoder.GetEncoders(type)) + { + yield return match; + } + } else if (decoder.CanEncode(type)) + { + yield return decoder; + } + } + } +} diff --git a/src/runtime/Codecs/RawProxyEncoder.cs b/src/runtime/Codecs/RawProxyEncoder.cs new file mode 100644 index 000000000..a1b6c52b3 --- /dev/null +++ b/src/runtime/Codecs/RawProxyEncoder.cs @@ -0,0 +1,21 @@ +using System; + +namespace Python.Runtime.Codecs +{ + /// + /// A .NET object encoder, that returns raw proxies (e.g. no conversion to Python types). + /// You must inherit from this class and override . + /// + [Obsolete(Util.UnstableApiMessage)] + public class RawProxyEncoder: IPyObjectEncoder + { + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return PyObject.FromManagedObject(value); + } + + public virtual bool CanEncode(Type type) => false; + } +} diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs new file mode 100644 index 000000000..a9ae33fe0 --- /dev/null +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -0,0 +1,135 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + [Obsolete(Util.UnstableApiMessage)] + public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder + { + TupleCodec() { } + public static TupleCodec Instance { get; } = new TupleCodec(); + + public bool CanEncode(Type type) + { + if (type == typeof(object) || type == typeof(TTuple)) return true; + return type.Namespace == typeof(TTuple).Namespace + // generic versions of tuples are named Tuple`TYPE_ARG_COUNT + && type.Name.StartsWith(typeof(TTuple).Name + '`'); + } + + public PyObject TryEncode(object value) + { + if (value == null) return null; + + var tupleType = value.GetType(); + if (tupleType == typeof(object)) return null; + if (!this.CanEncode(tupleType)) return null; + if (tupleType == typeof(TTuple)) return new PyTuple(); + + long fieldCount = tupleType.GetGenericArguments().Length; + var tuple = Runtime.PyTuple_New(fieldCount); + Exceptions.ErrorCheck(tuple); + int fieldIndex = 0; + foreach (FieldInfo field in tupleType.GetFields()) + { + var item = field.GetValue(value); + IntPtr pyItem = Converter.ToPython(item); + Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); + fieldIndex++; + } + return new PyTuple(tuple); + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == Runtime.PyTupleType && this.CanEncode(targetType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + value = default; + + if (!Runtime.PyTuple_Check(pyObj.Handle)) return false; + + if (typeof(T) == typeof(object)) + { + bool converted = Decode(pyObj, out object result); + if (converted) + { + value = (T)result; + return true; + } + + return false; + } + + var itemTypes = typeof(T).GetGenericArguments(); + long itemCount = Runtime.PyTuple_Size(pyObj.Handle); + if (itemTypes.Length != itemCount) return false; + + if (itemCount == 0) + { + value = (T)EmptyTuple; + return true; + } + + var elements = new object[itemCount]; + for (int itemIndex = 0; itemIndex < itemTypes.Length; itemIndex++) + { + IntPtr pyItem = Runtime.PyTuple_GetItem(pyObj.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, itemTypes[itemIndex], out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + } + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = (T)factory.Invoke(null, elements); + return true; + } + + static bool Decode(PyObject tuple, out object value) + { + long itemCount = Runtime.PyTuple_Size(tuple.Handle); + if (itemCount == 0) + { + value = EmptyTuple; + return true; + } + var elements = new object[itemCount]; + var itemTypes = new Type[itemCount]; + value = null; + for (int itemIndex = 0; itemIndex < elements.Length; itemIndex++) + { + var pyItem = Runtime.PyTuple_GetItem(tuple.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, typeof(object), out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + + itemTypes[itemIndex] = elements[itemIndex]?.GetType() ?? typeof(object); + } + + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = factory.Invoke(null, elements); + return true; + } + + static readonly MethodInfo[] tupleCreate = + typeof(TTuple).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Tuple.Create)) + .OrderBy(m => m.GetParameters().Length) + .ToArray(); + + static readonly object EmptyTuple = tupleCreate[0].Invoke(null, parameters: new object[0]); + + public static void Register() + { + PyObjectConversions.RegisterEncoder(Instance); + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index b51911816..0cbbbaba2 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -120,9 +120,7 @@ public static int GetUnicodeByteLength(IntPtr p) /// public static IntPtr Py3UnicodePy2StringtoPtr(string s) { - return Runtime.IsPython3 - ? Instance.MarshalManagedToNative(s) - : Marshal.StringToHGlobalAnsi(s); + return Instance.MarshalManagedToNative(s); } /// @@ -137,9 +135,7 @@ public static IntPtr Py3UnicodePy2StringtoPtr(string s) /// public static string PtrToPy3UnicodePy2String(IntPtr p) { - return Runtime.IsPython3 - ? PtrToStringUni(p) - : Marshal.PtrToStringAnsi(p); + return PtrToStringUni(p); } } diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs new file mode 100644 index 000000000..6e66232d0 --- /dev/null +++ b/src/runtime/NewReference.cs @@ -0,0 +1,70 @@ +namespace Python.Runtime +{ + using System; + using System.Diagnostics.Contracts; + + /// + /// Represents a reference to a Python object, that is tracked by Python's reference counting. + /// + [NonCopyable] + ref struct NewReference + { + IntPtr pointer; + + [Pure] + public static implicit operator BorrowedReference(in NewReference reference) + => new BorrowedReference(reference.pointer); + + /// + /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + /// + public PyObject MoveToPyObject() + { + if (this.IsNull()) throw new NullReferenceException(); + + var result = new PyObject(this.pointer); + this.pointer = IntPtr.Zero; + return result; + } + /// + /// Removes this reference to a Python object, and sets it to null. + /// + public void Dispose() + { + if (!this.IsNull()) + Runtime.XDecref(this.pointer); + this.pointer = IntPtr.Zero; + } + + /// + /// Creates from a raw pointer + /// + [Pure] + public static NewReference DangerousFromPointer(IntPtr pointer) + => new NewReference {pointer = pointer}; + + [Pure] + internal static IntPtr DangerousGetAddress(in NewReference reference) + => IsNull(reference) ? throw new NullReferenceException() : reference.pointer; + [Pure] + internal static bool IsNull(in NewReference reference) + => reference.pointer == IntPtr.Zero; + } + + /// + /// These members can not be directly in type, + /// because this is always passed by value, which we need to avoid. + /// (note this in NewReference vs the usual this NewReference) + /// + static class NewReferenceExtensions + { + /// Gets a raw pointer to the Python object + [Pure] + public static IntPtr DangerousGetAddress(this in NewReference reference) + => NewReference.DangerousGetAddress(reference); + [Pure] + public static bool IsNull(this in NewReference reference) + => NewReference.IsNull(reference); + } +} diff --git a/src/runtime/NonCopyableAttribute.cs b/src/runtime/NonCopyableAttribute.cs new file mode 100644 index 000000000..63d36ab42 --- /dev/null +++ b/src/runtime/NonCopyableAttribute.cs @@ -0,0 +1,6 @@ +namespace Python.Runtime +{ + using System; + [AttributeUsage(AttributeTargets.Struct)] + class NonCopyableAttribute : Attribute { } +} diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index a5d33c7ab..cab9df30b 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,7 +4,7 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("Python for .NET")] +[assembly: AssemblyTitle("Python.NET")] [assembly: AssemblyDescription("")] [assembly: AssemblyDefaultAlias("Python.Runtime.dll")] diff --git a/src/runtime/PyExportAttribute.cs b/src/runtime/PyExportAttribute.cs new file mode 100644 index 000000000..52a8be15d --- /dev/null +++ b/src/runtime/PyExportAttribute.cs @@ -0,0 +1,16 @@ +namespace Python.Runtime { + using System; + + /// + /// Controls visibility to Python for public .NET type or an entire assembly + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Delegate | AttributeTargets.Enum + | AttributeTargets.Interface | AttributeTargets.Struct | AttributeTargets.Assembly, + AllowMultiple = false, + Inherited = false)] + public class PyExportAttribute : Attribute + { + internal readonly bool Export; + public PyExportAttribute(bool export) { this.Export = export; } + } +} diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 794645994..d753a5dff 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -7,14 +7,21 @@ net45 Python.Runtime Python.Runtime - Python.Runtime - 2.4.0 - false - false - false - false - false - false + pythonnet + 2.5.0 + true + false + Codestin Search App + Copyright (c) 2006-2020 the contributors of the Python.NET project + Python and CLR (.NET and Mono) cross-platform language interop + pythonnet + https://github.com/pythonnet/pythonnet/blob/master/LICENSE + https://github.com/pythonnet/pythonnet + git + + python interop dynamic dlr Mono pinvoke + https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico + https://pythonnet.github.io/ bin\ false $(OutputPath)\$(AssemblyName).xml @@ -23,7 +30,7 @@ ..\..\ $(SolutionDir)\bin\ $(PythonBuildDir)\$(TargetFramework)\ - 6 + 7.3 True ..\pythonnet.snk $(PYTHONNET_DEFINE_CONSTANTS) @@ -35,7 +42,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON38 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -68,10 +75,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants) @@ -80,10 +87,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG @@ -122,6 +129,13 @@ + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index fc155ca91..7ee6b1d97 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,170 +1,186 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON37;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON37;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG - false - full - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + \ No newline at end of file diff --git a/src/runtime/ReferenceExtensions.cs b/src/runtime/ReferenceExtensions.cs new file mode 100644 index 000000000..8fa2731b7 --- /dev/null +++ b/src/runtime/ReferenceExtensions.cs @@ -0,0 +1,20 @@ +namespace Python.Runtime +{ + using System.Diagnostics.Contracts; + + static class ReferenceExtensions + { + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this in NewReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this BorrowedReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + } +} diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..eb21cddbb 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -3,8 +3,11 @@ namespace Python.Runtime { - internal class Util + internal static class Util { + internal const string UnstableApiMessage = + "This API is unstable, and might be changed or removed in the next minor release"; + internal static Int64 ReadCLong(IntPtr tp, int offset) { // On Windows, a C long is always 32 bits. @@ -29,5 +32,12 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags) Marshal.WriteInt64(type, offset, flags); } } + + /// + /// Null-coalesce: if parameter is not + /// , return it. Otherwise return . + /// + internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback) + => primary == IntPtr.Zero ? fallback : primary; } -} \ No newline at end of file +} diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..1ef318473 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -244,16 +244,5 @@ public static int sq_contains(IntPtr ob, IntPtr v) return 0; } - - - /// - /// Implements __len__ for array types. - /// - public static int mp_length(IntPtr ob) - { - var self = (CLRObject)GetManagedObject(ob); - var items = self.inst as Array; - return items.Length; - } } } diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..f5b0b3509 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -7,6 +6,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using System.Threading.Tasks; namespace Python.Runtime { @@ -25,18 +25,16 @@ internal class AssemblyManager // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - // unless LoaderOptimization.MultiDomain is used); // So for multidomain support it is better to have the dict. recreated for each app-domain initialization - private static ConcurrentDictionary> namespaces = - new ConcurrentDictionary>(); - //private static Dictionary> generics; - private static AssemblyLoadEventHandler lhandler; - private static ResolveEventHandler rhandler; + private static ConcurrentDictionary> namespaces; + private static ConcurrentDictionary assembliesNamesCache; + private static ConcurrentDictionary lookupTypeCache; + private static ConcurrentQueue assemblies; + private static int pendingAssemblies; // updated only under GIL? - private static Dictionary probed = new Dictionary(32); - - // modified from event handlers below, potentially triggered from different .NET threads - private static ConcurrentQueue assemblies; - internal static List pypath; + private static Dictionary probed; + private static List pypath; + private static Dictionary> filesInPath; private AssemblyManager() { @@ -49,32 +47,44 @@ private AssemblyManager() /// internal static void Initialize() { + namespaces = new ConcurrentDictionary>(); + assembliesNamesCache = new ConcurrentDictionary(); + lookupTypeCache = new ConcurrentDictionary(); + probed = new Dictionary(32); + assemblies = new ConcurrentQueue(); pypath = new List(16); + filesInPath = new Dictionary>(); AppDomain domain = AppDomain.CurrentDomain; - lhandler = new AssemblyLoadEventHandler(AssemblyLoadHandler); - domain.AssemblyLoad += lhandler; + domain.AssemblyLoad += AssemblyLoadHandler; + domain.AssemblyResolve += ResolveHandler; - rhandler = new ResolveEventHandler(ResolveHandler); - domain.AssemblyResolve += rhandler; - - Assembly[] items = domain.GetAssemblies(); - foreach (Assembly a in items) + foreach (var assembly in domain.GetAssemblies()) { try { - ScanAssembly(a); - assemblies.Enqueue(a); + LaunchAssemblyLoader(assembly); } catch (Exception ex) { - Debug.WriteLine("Error scanning assembly {0}. {1}", a, ex); + Debug.WriteLine($"Error scanning assembly {assembly}. {ex}"); } } - } + var safeCount = 0; + // lets wait until all assemblies are loaded + do + { + if (safeCount++ > 200) + { + throw new TimeoutException("Timeout while waiting for assemblies to load"); + } + + Thread.Sleep(50); + } while (pendingAssemblies > 0); + } /// /// Cleanup resources upon shutdown of the Python runtime. @@ -82,8 +92,8 @@ internal static void Initialize() internal static void Shutdown() { AppDomain domain = AppDomain.CurrentDomain; - domain.AssemblyLoad -= lhandler; - domain.AssemblyResolve -= rhandler; + domain.AssemblyLoad -= AssemblyLoadHandler; + domain.AssemblyResolve -= ResolveHandler; } @@ -94,13 +104,41 @@ internal static void Shutdown() /// so that we can know about assemblies that get loaded after the /// Python runtime is initialized. /// + /// Scanning assemblies here caused internal hangs when calling + /// private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) { Assembly assembly = args.LoadedAssembly; - assemblies.Enqueue(assembly); - ScanAssembly(assembly); + LaunchAssemblyLoader(assembly); } + /// + /// Launches a new task that will load the provided assembly + /// + private static void LaunchAssemblyLoader(Assembly assembly) + { + if (assembly != null) + { + Interlocked.Increment(ref pendingAssemblies); + Task.Factory.StartNew(() => + { + try + { + if (assembliesNamesCache.TryAdd(assembly.GetName().Name, assembly)) + { + assemblies.Enqueue(assembly); + ScanAssembly(assembly); + } + } + catch + { + // pass + } + + Interlocked.Decrement(ref pendingAssemblies); + }); + } + } /// /// Event handler for assembly resolve events. This is needed because @@ -112,12 +150,12 @@ private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) private static Assembly ResolveHandler(object ob, ResolveEventArgs args) { string name = args.Name.ToLower(); - foreach (Assembly a in assemblies) + foreach (var assembly in assemblies) { - string full = a.FullName.ToLower(); + var full = assembly.FullName.ToLower(); if (full.StartsWith(name)) { - return a; + return assembly; } } return LoadAssemblyPath(args.Name); @@ -138,24 +176,66 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); + var count = Runtime.PyList_Size(list); + var sep = Path.DirectorySeparatorChar; + if (count != pypath.Count) { pypath.Clear(); probed.Clear(); + // add first the current path + pypath.Add(""); for (var i = 0; i < count; i++) { - IntPtr item = Runtime.PyList_GetItem(list, i); + BorrowedReference item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { - pypath.Add(path); + pypath.Add(path == string.Empty ? path : path + sep); } } + + // for performance we will search for all files in each directory in the path once + Parallel.ForEach(pypath.Where(s => + { + try + { + lock (filesInPath) + { + // only search in directory if it exists and we haven't already analyzed it + return Directory.Exists(s) && !filesInPath.ContainsKey(s); + } + } + catch + { + // just in case, file operations can throw + } + return false; + }), path => + { + var container = new HashSet(); + try + { + foreach (var file in Directory.EnumerateFiles(path) + .Where(file => file.EndsWith(".dll") || file.EndsWith(".exe"))) + { + container.Add(Path.GetFileName(file)); + } + } + catch + { + // just in case, file operations can throw + } + + lock (filesInPath) + { + filesInPath[path] = container; + } + }); } } - /// /// Given an assembly name, try to find this assembly file using the /// PYTHONPATH. If not found, return null to indicate implicit load @@ -163,30 +243,17 @@ internal static void UpdatePath() /// public static string FindAssembly(string name) { - char sep = Path.DirectorySeparatorChar; - - foreach (string head in pypath) + foreach (var kvp in filesInPath) { - string path; - if (head == null || head.Length == 0) + var dll = $"{name}.dll"; + if (kvp.Value.Contains(dll)) { - path = name; + return kvp.Key + dll; } - else + var executable = $"{name}.exe"; + if (kvp.Value.Contains(executable)) { - path = head + sep + name; - } - - string temp = path + ".dll"; - if (File.Exists(temp)) - { - return temp; - } - - temp = path + ".exe"; - if (File.Exists(temp)) - { - return temp; + return kvp.Key + executable; } } return null; @@ -206,10 +273,7 @@ public static Assembly LoadAssembly(string name) } catch (Exception) { - //if (!(e is System.IO.FileNotFoundException)) - //{ - // throw; - //} + // ignored } return assembly; } @@ -220,7 +284,7 @@ public static Assembly LoadAssembly(string name) /// public static Assembly LoadAssemblyPath(string name) { - string path = FindAssembly(name); + var path = FindAssembly(name); Assembly assembly = null; if (path != null) { @@ -230,6 +294,7 @@ public static Assembly LoadAssemblyPath(string name) } catch (Exception) { + // ignored } } return assembly; @@ -257,6 +322,7 @@ public static Assembly LoadAssemblyFullPath(string name) } catch (Exception) { + // ignored } } } @@ -268,14 +334,8 @@ public static Assembly LoadAssemblyFullPath(string name) /// public static Assembly FindLoadedAssembly(string name) { - foreach (Assembly a in assemblies) - { - if (a.GetName().Name == name) - { - return a; - } - } - return null; + Assembly result; + return assembliesNamesCache.TryGetValue(name, out result) ? result : null; } /// @@ -312,10 +372,6 @@ public static bool LoadImplicit(string name, bool warn = true) { a = LoadAssemblyPath(s); } - if (a == null) - { - a = LoadAssembly(s); - } if (a != null && !assembliesSet.Contains(a)) { loaded = true; @@ -346,10 +402,15 @@ public static bool LoadImplicit(string name, bool warn = true) /// internal static void ScanAssembly(Assembly assembly) { + if (assembly.GetCustomAttribute()?.Export == false) + { + return; + } // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + Type[] types = assembly.GetTypes(); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -359,13 +420,13 @@ internal static void ScanAssembly(Assembly assembly) for (var n = 0; n < names.Length; n++) { s = n == 0 ? names[0] : s + "." + names[n]; - namespaces.TryAdd(s, new ConcurrentDictionary()); + namespaces.TryAdd(s, new ConcurrentDictionary()); } } if (ns != null) { - namespaces[ns].TryAdd(assembly, string.Empty); + namespaces[ns].TryAdd(assembly, 1); } if (ns != null && t.IsGenericTypeDefinition) @@ -452,38 +513,53 @@ public static List GetNames(string nsname) /// looking in the currently loaded assemblies for the named /// type. Returns null if the named type cannot be found. /// + [Obsolete("Use LookupTypes and handle name conflicts")] public static Type LookupType(string qname) { + Type type; + if (lookupTypeCache.TryGetValue(qname, out type)) + { + return type; + } foreach (Assembly assembly in assemblies) { - Type type = assembly.GetType(qname); - if (type != null) + type = assembly.GetType(qname); + if (type != null && IsExported(type)) { + lookupTypeCache[qname] = type; return type; } } return null; } + /// + /// Returns the objects for the given qualified name, + /// looking in the currently loaded assemblies for the named + /// type. + /// + public static IEnumerable LookupTypes(string qualifiedName) + => assemblies.Select(assembly => assembly.GetType(qualifiedName)).Where(type => type != null && IsExported(type)); + internal static Type[] GetTypes(Assembly a) { if (a.IsDynamic) { try { - return a.GetTypes(); + return a.GetTypes().Where(IsExported).ToArray(); } catch (ReflectionTypeLoadException exc) { // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); + return exc.Types.Where(x => x != null && IsExported(x)).ToArray(); } } else { try { - return a.GetExportedTypes(); + return a.GetExportedTypes().Where(IsExported).ToArray(); } catch (FileNotFoundException) { @@ -491,5 +567,7 @@ internal static Type[] GetTypes(Assembly a) } } } + + static bool IsExported(Type type) => type.GetCustomAttribute()?.Export != false; } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..43ec6ea72 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -246,6 +246,45 @@ public static IntPtr tp_str(IntPtr ob) } } + public static IntPtr tp_repr(IntPtr ob) + { + var co = GetManagedObject(ob) as CLRObject; + if (co == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + try + { + //if __repr__ is defined, use it + var instType = co.inst.GetType(); + System.Reflection.MethodInfo methodInfo = instType.GetMethod("__repr__"); + if (methodInfo != null && methodInfo.IsPublic) + { + var reprString = methodInfo.Invoke(co.inst, null) as string; + return Runtime.PyString_FromString(reprString); + } + + //otherwise use the standard object.__repr__(inst) + IntPtr args = Runtime.PyTuple_New(1); + Runtime.XIncref(ob); + Runtime.PyTuple_SetItem(args, 0, ob); + IntPtr reprFunc = Runtime.PyObject_GetAttrString(Runtime.PyBaseObjectType, "__repr__"); + var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero); + Runtime.XDecref(args); + Runtime.XDecref(reprFunc); + return output; + } + catch (Exception e) + { + if (e.InnerException != null) + { + e = e.InnerException; + } + Exceptions.SetError(e); + return IntPtr.Zero; + } + } + /// /// Standard dealloc implementation for instances of reflected types. @@ -253,7 +292,7 @@ public static IntPtr tp_str(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..af16b1359 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -877,7 +877,7 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..5ff222923 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -45,9 +45,8 @@ public static void Reset() /// internal static ClassBase GetClass(Type type) { - ClassBase cb = null; - cache.TryGetValue(type, out cb); - if (cb != null) + ClassBase cb; + if (cache.TryGetValue(type, out cb)) { return cb; } @@ -93,6 +92,11 @@ private static ClassBase CreateClass(Type type) impl = new ArrayObject(type); } + else if (type.IsKeyValuePairEnumerable()) + { + impl = new KeyValuePairEnumerableObject(type); + } + else if (type.IsInterface) { impl = new InterfaceObject(type); diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..5a0d73a42 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -31,9 +31,9 @@ internal ClassObject(Type tp) : base(tp) /// internal IntPtr GetDocString() { - MethodBase[] methods = binder.GetMethods(); + var methods = binder.GetMethods(); var str = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (str.Length > 0) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..29478ca3b 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,11 +14,11 @@ internal CLRObject(object ob, IntPtr tp) long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); + Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); } } @@ -29,19 +29,23 @@ internal CLRObject(object ob, IntPtr tp) gcHandle = gc; inst = ob; - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + // for performance before calling SetArgsAndCause() lets check if we are an exception + if (inst is Exception) + { + // Fix the BaseException args (and __cause__ in case of Python 3) + // slot if wrapping a CLR exception + Exceptions.SetArgsAndCause(py); + } } - internal static CLRObject GetInstance(object ob, IntPtr pyType) + static CLRObject GetInstance(object ob, IntPtr pyType) { return new CLRObject(ob, pyType); } - internal static CLRObject GetInstance(object ob) + static CLRObject GetInstance(object ob) { ClassBase cc = ClassManager.GetClass(ob.GetType()); return GetInstance(ob, cc.tpHandle); diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 1fc541920..0f9806c0e 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Text; namespace Python.Runtime { @@ -93,7 +94,15 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) if (binding == null) { - Exceptions.SetError(Exceptions.TypeError, "no constructor matches given arguments"); + var errorMessage = new StringBuilder("No constructor matches given arguments"); + if (info != null && info.IsConstructor && info.DeclaringType != null) + { + errorMessage.Append(" for ").Append(info.DeclaringType.Name); + } + + errorMessage.Append(": "); + AppendArgumentTypes(to: errorMessage, args); + Exceptions.SetError(Exceptions.TypeError, errorMessage.ToString()); return null; } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..de7284031 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -119,10 +119,10 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XIncref(self.repr); return self.repr; } - MethodBase[] methods = self.ctorBinder.GetMethods(); + var methods = self.ctorBinder.GetMethods(); string name = self.type.FullName; var doc = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (doc.Length > 0) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..4506c1b51 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,11 +1,10 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; using System.ComponentModel; +using System.Collections.Generic; namespace Python.Runtime { @@ -31,6 +30,9 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -46,6 +48,31 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -72,6 +99,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -86,9 +116,6 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == int32Type) return Runtime.PyIntType; - if (op == int64Type && Runtime.IsPython2) - return Runtime.PyLongType; - if (op == int64Type) return Runtime.PyIntType; @@ -101,6 +128,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyDecimalType; + return IntPtr.Zero; } @@ -116,6 +146,23 @@ internal static IntPtr ToPython(T value) return ToPython(value, typeof(T)); } + private static readonly Func IsTransparentProxy = GetIsTransparentProxy(); + + private static bool Never(object _) => false; + + private static Func GetIsTransparentProxy() + { + var remoting = typeof(int).Assembly.GetType("System.Runtime.Remoting.RemotingServices"); + if (remoting is null) return Never; + + var isProxy = remoting.GetMethod("IsTransparentProxy", new[] { typeof(object) }); + if (isProxy is null) return Never; + + return (Func)Delegate.CreateDelegate( + typeof(Func), isProxy, + throwOnBindFailure: true); + } + internal static IntPtr ToPython(object value, Type type) { if (value is PyObject) @@ -135,19 +182,31 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { - using (var resultlist = new PyList()) + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) + { + var encoded = PyObjectConversions.TryEncode(value, type); + + if (encoded != null) { + result = encoded.Handle; + Runtime.XIncref(result); + return result; + } + } + + var list = value as IList; + if (list != null && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) + { + using (var resultList = new PyList()) { - foreach (object o in (IEnumerable)value) + for (var i = 0; i < list.Count; i++) { - using (var p = new PyObject(ToPython(o, o?.GetType()))) + using (var p = list[i].ToPython()) { - resultlist.Append(p); + resultList.Append(p); } } - Runtime.XIncref(resultlist.Handle); - return resultlist.Handle; + Runtime.XIncref(resultList.Handle); + return resultList.Handle; } } @@ -156,15 +215,8 @@ internal static IntPtr ToPython(object value, Type type) var pyderived = value as IPythonDerivedType; if (null != pyderived) { - #if NETSTANDARD - return ClassDerivedObject.ToPython(pyderived); - #else - // if object is remote don't do this - if (!System.Runtime.Remoting.RemotingServices.IsTransparentProxy(pyderived)) - { + if (!IsTransparentProxy(pyderived)) return ClassDerivedObject.ToPython(pyderived); - } - #endif } // hmm - from Python, we almost never care what the declared @@ -178,6 +230,17 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -230,6 +293,40 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); + + case TypeCode.DateTime: + var datetime = (DateTime)value; + + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; + default: if (value is IEnumerable) { @@ -251,6 +348,18 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; + } + /// /// In a few situations, we don't have any advisory type information @@ -294,6 +403,15 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return true; } + if (obType.IsGenericType && Runtime.PyObject_TYPE(value) == Runtime.PyListType) + { + var typeDefinition = obType.GetGenericTypeDefinition(); + if (typeDefinition == typeof(List<>)) + { + return ToList(value, obType, out result, setError); + } + } + // Common case: if the Python value is a wrapped managed object // instance, just return the wrapped object. ManagedType mt = ManagedType.GetManagedObject(value); @@ -309,6 +427,16 @@ internal static bool ToManagedValue(IntPtr value, Type obType, result = tmp; return true; } + + var type = tmp.GetType(); + // check implicit conversions that receive tmp type and return obType + var conversionMethod = type.GetMethod("op_Implicit", new[] { type }); + if (conversionMethod != null && conversionMethod.ReturnType == obType) + { + result = conversionMethod.Invoke(null, new[] { tmp }); + return true; + } + Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); return false; } @@ -377,17 +505,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToPrimitive(value, doubleType, out result, setError); } - if (Runtime.PySequence_Check(value)) + // give custom codecs a chance to take over conversion of sequences + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { - return ToArray(value, typeof(object[]), out result, setError); + return true; } - if (setError) + if (Runtime.PySequence_Check(value)) { - Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Object"); + return ToArray(value, typeof(object[]), out result, setError); } - return false; + Runtime.XIncref(value); // PyObject() assumes ownership + result = new PyObject(value); + return true; } // Conversion to 'Type' is done using the same mappings as above for objects. @@ -437,9 +569,42 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] {result.GetType()}); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] {result}); + } + + return opImplicit != null; + } + } + + TypeCode typeCode = Type.GetTypeCode(obType); + if (typeCode == TypeCode.Object) + { + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + return ToPrimitive(value, obType, out result, setError); } + internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + /// /// Convert a Python value to an instance of a primitive managed type. /// @@ -453,6 +618,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -463,63 +654,35 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo return true; case TypeCode.Int32: - // Trickery to support 64-bit platforms. - if (Runtime.IsPython2 && Runtime.Is32Bit) + // Python3 always use PyLong API + op = Runtime.PyNumber_Long(value); + if (op == IntPtr.Zero) { - op = Runtime.PyNumber_Int(value); - - // As of Python 2.3, large ints magically convert :( - if (Runtime.PyLong_Check(op)) + Exceptions.Clear(); + if (Exceptions.ExceptionMatches(overflow)) { - Runtime.XDecref(op); goto overflow; } - - if (op == IntPtr.Zero) - { - if (Exceptions.ExceptionMatches(overflow)) - { - goto overflow; - } - goto type_error; - } - ival = (int)Runtime.PyInt_AsLong(op); - Runtime.XDecref(op); - result = ival; - return true; + goto type_error; + } + long ll = (long)Runtime.PyLong_AsLongLong(op); + Runtime.XDecref(op); + if (ll == -1 && Exceptions.ErrorOccurred()) + { + goto overflow; } - else // Python3 always use PyLong API + if (ll > Int32.MaxValue || ll < Int32.MinValue) { - op = Runtime.PyNumber_Long(value); - if (op == IntPtr.Zero) - { - Exceptions.Clear(); - if (Exceptions.ExceptionMatches(overflow)) - { - goto overflow; - } - goto type_error; - } - long ll = (long)Runtime.PyLong_AsLongLong(op); - Runtime.XDecref(op); - if (ll == -1 && Exceptions.ErrorOccurred()) - { - goto overflow; - } - if (ll > Int32.MaxValue || ll < Int32.MinValue) - { - goto overflow; - } - result = (int)ll; - return true; + goto overflow; } + result = (int)ll; + return true; case TypeCode.Boolean: result = Runtime.PyObject_IsTrue(value) != 0; return true; case TypeCode.Byte: -#if PYTHON3 if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { if (Runtime.PyBytes_Size(value) == 1) @@ -530,18 +693,6 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) - { - op = Runtime.PyString_AsString(value); - result = (byte)Marshal.ReadByte(op); - return true; - } - goto type_error; - } -#endif op = Runtime.PyNumber_Int(value); if (op == IntPtr.Zero) @@ -564,7 +715,6 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo return true; case TypeCode.SByte: -#if PYTHON3 if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { if (Runtime.PyBytes_Size(value) == 1) @@ -575,18 +725,6 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) - { - op = Runtime.PyString_AsString(value); - result = (sbyte)Marshal.ReadByte(op); - return true; - } - goto type_error; - } -#endif op = Runtime.PyNumber_Int(value); if (op == IntPtr.Zero) @@ -609,7 +747,6 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo return true; case TypeCode.Char: -#if PYTHON3 if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { if (Runtime.PyBytes_Size(value) == 1) @@ -620,18 +757,6 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } -#elif PYTHON2 - if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) - { - if (Runtime.PyString_Size(value) == 1) - { - op = Runtime.PyString_AsString(value); - result = (char)Marshal.ReadByte(op); - return true; - } - goto type_error; - } -#endif else if (Runtime.PyObject_TypeCheck(value, Runtime.PyUnicodeType)) { if (Runtime.PyUnicode_GetSize(value) == 1) @@ -728,7 +853,20 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } - uint ui = (uint)Runtime.PyLong_AsUnsignedLong(op); + + uint ui; + try + { + ui = Convert.ToUInt32(Runtime.PyLong_AsUnsignedLong(op)); + } + catch (OverflowException) + { + // Probably wasn't an overflow in python but was in C# (e.g. if cpython + // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in + // PyLong_AsUnsignedLong) + Runtime.XDecref(op); + goto overflow; + } if (Exceptions.ErrorOccurred()) { @@ -802,6 +940,30 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number | NumberStyles.AllowExponent, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; + return true; } @@ -837,15 +999,84 @@ private static void SetConversionError(IntPtr value, Type target) /// /// Convert a Python value to a correctly typed managed array instance. - /// The Python value must support the Python sequence protocol and the + /// The Python value must support the Python iterator protocol or and the /// items in the sequence must be convertible to the target array type. /// private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { + result = null; + IntPtr IterObject = Runtime.PyObject_GetIter(value); + + if (IterObject == IntPtr.Zero) + { + if (setError) + { + SetConversionError(value, obType); + } + return false; + } + Type elementType = obType.GetElementType(); + var listType = typeof(List<>); + var constructedListType = listType.MakeGenericType(elementType); + + IList list; + if (Runtime.PySequence_Check(value)) + { + var len = Runtime.PySequence_Size(value); + list = (IList) Activator.CreateInstance(constructedListType, len); + } + else + { + list = (IList) Activator.CreateInstance(constructedListType); + } + + IntPtr item; + + while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) + { + object obj; + + if (!Converter.ToManaged(item, elementType, out obj, true)) + { + Runtime.XDecref(item); + return false; + } + + list.Add(obj); + Runtime.XDecref(item); + } + Runtime.XDecref(IterObject); + + var items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); + + result = items; + return true; + } + + /// + /// Convert a Python value to a correctly typed managed list instance. + /// The Python value must support the Python sequence protocol and the + /// items in the sequence must be convertible to the target list type. + /// + private static bool ToList(IntPtr value, Type obType, out object result, bool setError) + { + var elementType = obType.GetGenericArguments()[0]; var size = Runtime.PySequence_Size(value); - result = null; + result = Activator.CreateInstance(obType, args: size); + var resultList = (IList)result; + return ApplyActionToPySequence(value, obType, setError, size, elementType, o => resultList.Add(o)); + } + + private static bool ApplyActionToPySequence(IntPtr value, + Type obType, + bool setError, + long size, + Type elementType, + Action action) + { if (size < 0) { if (setError) @@ -855,12 +1086,9 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - Array items = Array.CreateInstance(elementType, size); - - // XXX - is there a better way to unwrap this if it is a real array? for (var i = 0; i < size; i++) { - object obj = null; + object obj; IntPtr item = Runtime.PySequence_GetItem(value, i); if (item == IntPtr.Zero) { @@ -877,15 +1105,12 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + action(obj); Runtime.XDecref(item); } - - result = items; return true; } - /// /// Convert a Python value to a correctly typed managed enum instance. /// diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs new file mode 100644 index 000000000..667fc6f00 --- /dev/null +++ b/src/runtime/converterextensions.cs @@ -0,0 +1,192 @@ +namespace Python.Runtime +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using Python.Runtime.Codecs; + + /// + /// Defines conversion to CLR types (unmarshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectDecoder + { + /// + /// Checks if this decoder can decode from to + /// + bool CanDecode(PyObject objectType, Type targetType); + /// + /// Attempts do decode into a variable of specified type + /// + /// CLR type to decode into + /// Object to decode + /// The variable, that will receive decoding result + /// + bool TryDecode(PyObject pyObj, out T value); + } + + /// + /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectEncoder + { + /// + /// Checks if encoder can encode CLR objects of specified type + /// + bool CanEncode(Type type); + /// + /// Attempts to encode CLR object into Python object + /// + PyObject TryEncode(object value); + } + + /// + /// This class allows to register additional marshalling codecs. + /// Python.NET will pick suitable encoder/decoder registered first + /// + [Obsolete(Util.UnstableApiMessage)] + public static class PyObjectConversions + { + static readonly DecoderGroup decoders = new DecoderGroup(); + static readonly EncoderGroup encoders = new EncoderGroup(); + + /// + /// Registers specified encoder (marshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterEncoder(IPyObjectEncoder encoder) + { + if (encoder == null) throw new ArgumentNullException(nameof(encoder)); + + lock (encoders) + { + encoders.Add(encoder); + } + } + + /// + /// Registers specified decoder (unmarshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterDecoder(IPyObjectDecoder decoder) + { + if (decoder == null) throw new ArgumentNullException(nameof(decoder)); + + lock (decoders) + { + decoders.Add(decoder); + } + } + + #region Encoding + internal static PyObject TryEncode(object obj, Type type) + { + if (obj == null) throw new ArgumentNullException(nameof(obj)); + if (type == null) throw new ArgumentNullException(nameof(type)); + + foreach (var encoder in clrToPython.GetOrAdd(type, GetEncoders)) + { + var result = encoder.TryEncode(obj); + if (result != null) return result; + } + + return null; + } + + static readonly ConcurrentDictionary + clrToPython = new ConcurrentDictionary(); + static IPyObjectEncoder[] GetEncoders(Type type) + { + lock (encoders) + { + return encoders.GetEncoders(type).ToArray(); + } + } + #endregion + + #region Decoding + static readonly ConcurrentDictionary + pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) + { + if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); + if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (targetType == null) throw new ArgumentNullException(nameof(targetType)); + + var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + result = null; + if (decoder == null) return false; + return decoder.Invoke(pyHandle, out result); + } + + static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) + { + IPyObjectDecoder decoder; + using (var pyType = new PyObject(Runtime.SelfIncRef(sourceType))) + { + lock (decoders) + { + decoder = decoders.GetDecoder(pyType, targetType); + if (decoder == null) return null; + } + } + + var decode = genericDecode.MakeGenericMethod(targetType); + + bool TryDecode(IntPtr pyHandle, out object result) + { + var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle)); + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + if (!success) + { + pyObj.Dispose(); + } + + result = @params[1]; + return success; + } + + return TryDecode; + } + + static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); + + #endregion + + internal static void Reset() + { + lock (encoders) + lock (decoders) + { + clrToPython.Clear(); + pythonToClr.Clear(); + encoders.Clear(); + decoders.Clear(); + } + } + + struct TypePair : IEquatable + { + internal readonly IntPtr PyType; + internal readonly Type ClrType; + + public TypePair(IntPtr pyType, Type clrType) + { + this.PyType = pyType; + this.ClrType = clrType; + } + + public override int GetHashCode() + => this.ClrType.GetHashCode() ^ this.PyType.GetHashCode(); + + public bool Equals(TypePair other) + => this.PyType == other.PyType && this.ClrType == other.ClrType; + + public override bool Equals(object obj) => obj is TypePair other && this.Equals(other); + } + } +} diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..bd8f1ee4c 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher + public class Dispatcher : IPyDisposable { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + if (_finalized || _disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); + } - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + public void Dispose() + { + if (_disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) @@ -267,6 +276,11 @@ public object TrueDispatch(ArrayList args) Runtime.XDecref(op); return result; } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { target }; + } } diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index e1103cbc7..c9aad9898 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -96,7 +96,6 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) /// /// Implements __cmp__ for reflected delegate types. /// -#if PYTHON3 // TODO: Doesn't PY2 implement tp_richcompare too? public new static IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { if (op != Runtime.Py_EQ && op != Runtime.Py_NE) @@ -126,13 +125,5 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) Runtime.XIncref(pyfalse); return pyfalse; } -#elif PYTHON2 - public static int tp_compare(IntPtr ob, IntPtr other) - { - Delegate d1 = GetTrueDelegate(ob); - Delegate d2 = GetTrueDelegate(other); - return d1 == d2 ? 0 : -1; - } -#endif } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..e5efecbcf 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -36,6 +36,29 @@ internal static Exception ToException(IntPtr ob) return e; } + /// + /// Exception __repr__ implementation + /// + public new static IntPtr tp_repr(IntPtr ob) + { + Exception e = ToException(ob); + if (e == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + string name = e.GetType().Name; + string message; + if (e.Message != String.Empty) + { + message = String.Format("{0}('{1}')", name, e.Message); + } + else + { + message = String.Format("{0}()", name); + } + return Runtime.PyUnicode_FromString(message); + } + /// /// Exception __str__ implementation /// @@ -80,7 +103,7 @@ private Exceptions() /// internal static void Initialize() { - string exceptionsModuleName = Runtime.IsPython3 ? "builtins" : "exceptions"; + string exceptionsModuleName = "builtins"; exceptions_module = Runtime.PyImport_ImportModule(exceptionsModuleName); Exceptions.ErrorCheck(exceptions_module); @@ -157,13 +180,11 @@ internal static void SetArgsAndCause(IntPtr ob) Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); -#if PYTHON3 if (e.InnerException != null) { IntPtr cause = CLRObject.GetInstHandle(e.InnerException); Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); } -#endif } /// @@ -363,9 +384,6 @@ puplic static variables on the Exceptions class filled in from public static IntPtr Exception; public static IntPtr StopIteration; public static IntPtr GeneratorExit; -#if PYTHON2 - public static IntPtr StandardError; -#endif public static IntPtr ArithmeticError; public static IntPtr LookupError; diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..6585180c1 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -38,6 +38,7 @@ public ExtensionType() Runtime.PyObject_GC_UnTrack(py); + // Steals a ref to tpHandle. tpHandle = tp; pyHandle = py; gcHandle = gc; @@ -50,7 +51,7 @@ public ExtensionType() public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); + // Not necessary for decref of `tpHandle`. self.gcHandle.Free(); } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..ba562cc26 --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; + + public int Threshold { get; set; } + public bool Enable { get; set; } + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private int _throttled; + + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"{name} may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + public event IncorrectRefCntHandler IncorrectRefCntResolver; + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + + private Finalizer() + { + Enable = true; + Threshold = 200; + } + + [Obsolete("forceDispose parameter is unused. All objects are disposed regardless.")] + public void Collect(bool forceDispose) => this.DisposeAll(); + public void Collect() => this.DisposeAll(); + + internal void ThrottledCollect() + { + _throttled = unchecked(this._throttled + 1); + if (!Enable || _throttled < Threshold) return; + _throttled = 0; + this.Collect(); + } + + public List GetCollectedObjects() + { + return _objQueue.Select(T => new WeakReference(T)).ToList(); + } + + internal void AddFinalizedObject(IPyDisposable obj) + { + if (!Enable) + { + return; + } + +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + this._objQueue.Enqueue(obj); + } + } + + internal static void Shutdown() + { + Instance.DisposeAll(); + } + + private void DisposeAll() + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + try + { + obj.Dispose(); + } + catch (Exception e) + { + var handler = ErrorHandler; + if (handler is null) + { + throw new FinalizationException( + "Python object finalization failed", + disposable: obj, innerException: e); + } + + handler.Invoke(this, new ErrorArgs() + { + Error = e + }); + } + } + } + } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + IntPtr[] handles = obj.GetTrackedHandles(); + foreach (var handle in handles) + { + if (handle == IntPtr.Zero) + { + continue; + } + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResolver != null) + { + var funcList = IncorrectRefCntResolver.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif + } + + public class FinalizationException : Exception + { + public IPyDisposable Disposable { get; } + + public FinalizationException(string message, IPyDisposable disposable, Exception innerException) + : base(message, innerException) + { + this.Disposable = disposable ?? throw new ArgumentNullException(nameof(disposable)); + } + } +} diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..05e61cd65 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -31,32 +31,33 @@ public static void Reset() /// internal static void Register(Type t) { - if (null == t.Namespace || null == t.Name) + lock (mapping) { - return; - } + if (null == t.Namespace || null == t.Name) + { + return; + } - Dictionary> nsmap = null; - mapping.TryGetValue(t.Namespace, out nsmap); - if (nsmap == null) - { - nsmap = new Dictionary>(); - mapping[t.Namespace] = nsmap; - } - string basename = t.Name; - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } - List gnames = null; - nsmap.TryGetValue(basename, out gnames); - if (gnames == null) - { - gnames = new List(); - nsmap[basename] = gnames; + Dictionary> nsmap; + if (!mapping.TryGetValue(t.Namespace, out nsmap)) + { + nsmap = new Dictionary>(); + mapping[t.Namespace] = nsmap; + } + string basename = t.Name; + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } + List gnames; + if (!nsmap.TryGetValue(basename, out gnames)) + { + gnames = new List(); + nsmap[basename] = gnames; + } + gnames.Add(t.Name); } - gnames.Add(t.Name); } /// @@ -64,18 +65,20 @@ internal static void Register(Type t) /// public static List GetGenericBaseNames(string ns) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - var names = new List(); - foreach (string key in nsmap.Keys) + lock (mapping) { - names.Add(key); + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + var names = new List(); + foreach (string key in nsmap.Keys) + { + names.Add(key); + } + return names; } - return names; } /// @@ -105,38 +108,38 @@ public static List GenericsForType(Type t) public static List GenericsByName(string ns, string basename) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + lock (mapping) { - return null; - } + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } - List names = null; - nsmap.TryGetValue(basename, out names); - if (names == null) - { - return null; - } + List names; + if (!nsmap.TryGetValue(basename, out names)) + { + return null; + } - var result = new List(); - foreach (string name in names) - { - string qname = ns + "." + name; - Type o = AssemblyManager.LookupType(qname); - if (o != null) + var result = new List(); + foreach (string name in names) { - result.Add(o); + string qname = ns + "." + name; + Type o = AssemblyManager.LookupType(qname); + if (o != null) + { + result.Add(o); + } } + return result; } - - return result; } /// @@ -144,17 +147,19 @@ public static List GenericsByName(string ns, string basename) /// public static string GenericNameForBaseName(string ns, string name) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - List gnames = null; - nsmap.TryGetValue(name, out gnames); - if (gnames?.Count > 0) + lock (mapping) { - return gnames[0]; + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + List gnames = null; + nsmap.TryGetValue(name, out gnames); + if (gnames?.Count > 0) + { + return gnames[0]; + } } return null; } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..4c797ff8d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -13,7 +13,6 @@ internal class ImportHook private static MethodWrapper hook; private static IntPtr py_clr_module; -#if PYTHON3 private static IntPtr module_def = IntPtr.Zero; internal static void InitializeModuleDef() @@ -23,31 +22,64 @@ internal static void InitializeModuleDef() module_def = ModuleDefOffset.AllocModuleDef("clr"); } } -#endif + + internal static void ReleaseModuleDef() + { + if (module_def == IntPtr.Zero) + { + return; + } + ModuleDefOffset.FreeModuleDef(module_def); + module_def = IntPtr.Zero; + } /// - /// Initialization performed on startup of the Python runtime. + /// Initialize just the __import__ hook itself. /// - internal static void Initialize() + static void InitImport() { - // Initialize the Python <--> CLR module hook. We replace the - // built-in Python __import__ with our own. This isn't ideal, - // but it provides the most "Pythonic" way of dealing with CLR - // modules (Python doesn't provide a way to emulate packages). - IntPtr dict = Runtime.PyImport_GetModuleDict(); + // We replace the built-in Python __import__ with our own: first + // look in CLR modules, then if we don't find any call the default + // Python __import__. + IntPtr builtins = Runtime.GetBuiltins(); + py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); + PythonException.ThrowIfIsNull(py_import); - IntPtr mod = Runtime.IsPython3 - ? Runtime.PyImport_ImportModule("builtins") - : Runtime.PyDict_GetItemString(dict, "__builtin__"); - - py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); - Runtime.XDecref(hook.ptr); + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr); + PythonException.ThrowIfIsNotZero(res); + + Runtime.XDecref(builtins); + } + + /// + /// Restore the __import__ hook. + /// + static void RestoreImport() + { + IntPtr builtins = Runtime.GetBuiltins(); + + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); + PythonException.ThrowIfIsNotZero(res); + Runtime.XDecref(py_import); + py_import = IntPtr.Zero; + + hook.Release(); + hook = null; + Runtime.XDecref(builtins); + } + + /// + /// Initialization performed on startup of the Python runtime. + /// + internal static void Initialize() + { + InitImport(); + + // Initialize the clr module and tell Python about it. root = new CLRModule(); -#if PYTHON3 // create a python module with the same methods as the clr module-like object InitializeModuleDef(); py_clr_module = Runtime.PyModule_Create2(module_def, 3); @@ -58,10 +90,7 @@ internal static void Initialize() clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); Runtime.PyDict_Update(mod_dict, clr_dict); -#elif PYTHON2 - Runtime.XIncref(root.pyHandle); // we are using the module two times - py_clr_module = root.pyHandle; // Alias handle for PY2/PY3 -#endif + IntPtr dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); } @@ -72,12 +101,24 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) { - Runtime.XDecref(py_clr_module); - Runtime.XDecref(root.pyHandle); - Runtime.XDecref(py_import); + return; } + + RestoreImport(); + + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; + if (shouldFreeDef) + { + ReleaseModuleDef(); + } + + Runtime.XDecref(root.pyHandle); + root = null; + CLRModule.Reset(); } /// @@ -87,13 +128,6 @@ public static IntPtr GetCLRModule(IntPtr? fromList = null) { root.InitializePreload(); - if (Runtime.IsPython2) - { - Runtime.XIncref(py_clr_module); - return py_clr_module; - } - - // Python 3 // update the module dictionary with the contents of the root dictionary root.LoadNames(); IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module); diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..dbb33df8f 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,15 +56,14 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - var pynargs = Runtime.PyTuple_Size(args); - MethodBase[] methods = SetterBinder.GetMethods(); - if (methods.Length == 0) + var methods = SetterBinder.GetMethods(); + if (methods.Count == 0) { return false; } + var pynargs = Runtime.PyTuple_Size(args); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var pi = methods[0].ParameterInfo; // need to subtract one for the value int clrnargs = pi.Length - 1; if (pynargs == clrnargs || pynargs > clrnargs) @@ -98,9 +97,8 @@ internal IntPtr GetDefaultArgs(IntPtr args) var pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple - MethodBase[] methods = SetterBinder.GetMethods(); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var methods = SetterBinder.GetMethods(); + ParameterInfo[] pi = methods[0].ParameterInfo; int clrnargs = pi.Length - 1; IntPtr defaultArgs = Runtime.PyTuple_New(clrnargs - pynargs); for (var i = 0; i < clrnargs - pynargs; i++) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..027aee47b 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,9 +1,11 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Reflection; using System.Text; +using System.Collections.Generic; namespace Python.Runtime { @@ -67,11 +69,47 @@ public ModulePropertyAttribute() } } + internal static class ManagedDataOffsets + { + static ManagedDataOffsets() + { + FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset + size = fi.Length * IntPtr.Size; + } + + public static readonly int ob_data; + public static readonly int ob_dict; + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + return typeSize; + } + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + ob_data; + } + + public static int DictOffset(IntPtr type) + { + return BaseOffset(type) + ob_dict; + } + + public static int Size { get { return size; } } + + private static readonly int size; + } + + internal static class OriginalObjectOffsets { - static ObjectOffset() + static OriginalObjectOffsets() { int size = IntPtr.Size; var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD @@ -82,42 +120,58 @@ static ObjectOffset() #endif ob_refcnt = (n + 0) * size; ob_type = (n + 1) * size; - ob_dict = (n + 2) * size; - ob_data = (n + 3) * size; } - public static int magic(IntPtr ob) + public static int Size { get { return size; } } + + private static readonly int size = +#if PYTHON_WITH_PYDEBUG + 4 * IntPtr.Size; +#else + 2 * IntPtr.Size; +#endif + +#if PYTHON_WITH_PYDEBUG + public static int _ob_next; + public static int _ob_prev; +#endif + public static int ob_refcnt; + public static int ob_type; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ObjectOffset + { + static ObjectOffset() + { +#if PYTHON_WITH_PYDEBUG + _ob_next = OriginalObjectOffsets._ob_next; + _ob_prev = OriginalObjectOffsets._ob_prev; +#endif + ob_refcnt = OriginalObjectOffsets.ob_refcnt; + ob_type = OriginalObjectOffsets.ob_type; + + size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + } + + public static int magic(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_data; - } - return ob_data; + return ManagedDataOffsets.DataOffset(type); } - public static int DictOffset(IntPtr ob) + public static int TypeDictOffset(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_dict; - } - return ob_dict; + return ManagedDataOffsets.DictOffset(type); } - public static int Size(IntPtr ob) + public static int Size(IntPtr pyType) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(pyType)) { return ExceptionOffset.Size(); } -#if PYTHON_WITH_PYDEBUG - return 6 * IntPtr.Size; -#else - return 4 * IntPtr.Size; -#endif + + return size; } #if PYTHON_WITH_PYDEBUG @@ -126,8 +180,16 @@ public static int Size(IntPtr ob) #endif public static int ob_refcnt; public static int ob_type; - private static int ob_dict; - private static int ob_data; + private static readonly int size; + + private static bool IsException(IntPtr pyObject) + { + var type = Runtime.PyObject_TYPE(pyObject); + + return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) + || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -136,39 +198,30 @@ internal class ExceptionOffset static ExceptionOffset() { Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; + FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); } - } - public static int Size() - { - return ob_data + IntPtr.Size; + size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } + public static int Size() { return size; } + // PyException_HEAD // (start after PyObject_HEAD) public static int dict = 0; public static int args = 0; -#if PYTHON2 - public static int message = 0; -#elif PYTHON3 public static int traceback = 0; public static int context = 0; public static int cause = 0; public static int suppress_context = 0; -#endif - // extra c# data - public static int ob_dict; - public static int ob_data; + private static readonly int size; } -#if PYTHON3 [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] internal class BytesOffset { @@ -227,7 +280,7 @@ public static IntPtr AllocModuleDef(string modulename) byte[] ascii = Encoding.ASCII.GetBytes(modulename); int size = name + ascii.Length + 1; IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i < m_free; i += IntPtr.Size) + for (int i = 0; i <= m_free; i += IntPtr.Size) Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); @@ -259,7 +312,6 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } -#endif // PYTHON3 /// /// TypeFlags(): The actual bit values for the Type Flags stored @@ -269,17 +321,6 @@ public static void FreeModuleDef(IntPtr ptr) /// internal class TypeFlags { -#if PYTHON2 // these flags were removed in Python 3 - public static int HaveGetCharBuffer = (1 << 0); - public static int HaveSequenceIn = (1 << 1); - public static int GC = 0; - public static int HaveInPlaceOps = (1 << 3); - public static int CheckTypes = (1 << 4); - public static int HaveRichCompare = (1 << 5); - public static int HaveWeakRefs = (1 << 6); - public static int HaveIter = (1 << 7); - public static int HaveClass = (1 << 8); -#endif public static int HeapType = (1 << 9); public static int BaseType = (1 << 10); public static int Ready = (1 << 12); @@ -307,23 +348,9 @@ internal class TypeFlags public static int BaseExceptionSubclass = (1 << 30); public static int TypeSubclass = (1 << 31); -#if PYTHON2 // Default flags for Python 2 - public static int Default = ( - HaveGetCharBuffer | - HaveSequenceIn | - HaveInPlaceOps | - HaveRichCompare | - HaveWeakRefs | - HaveIter | - HaveClass | - HaveStacklessExtension | - HaveIndex | - 0); -#elif PYTHON3 // Default flags for Python 3 public static int Default = ( HaveStacklessExtension | HaveVersionTag); -#endif } @@ -334,7 +361,7 @@ internal class TypeFlags internal class Interop { - private static ArrayList keepAlive; + private static List keepAlive; private static Hashtable pmap; static Interop() @@ -351,8 +378,7 @@ static Interop() p[item.Name] = item; } - keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); + keepAlive = new List(); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -382,9 +408,6 @@ static Interop() pmap["nb_add"] = p["BinaryFunc"]; pmap["nb_subtract"] = p["BinaryFunc"]; pmap["nb_multiply"] = p["BinaryFunc"]; -#if PYTHON2 - pmap["nb_divide"] = p["BinaryFunc"]; -#endif pmap["nb_remainder"] = p["BinaryFunc"]; pmap["nb_divmod"] = p["BinaryFunc"]; pmap["nb_power"] = p["TernaryFunc"]; @@ -407,9 +430,6 @@ static Interop() pmap["nb_inplace_add"] = p["BinaryFunc"]; pmap["nb_inplace_subtract"] = p["BinaryFunc"]; pmap["nb_inplace_multiply"] = p["BinaryFunc"]; -#if PYTHON2 - pmap["nb_inplace_divide"] = p["BinaryFunc"]; -#endif pmap["nb_inplace_remainder"] = p["BinaryFunc"]; pmap["nb_inplace_power"] = p["TernaryFunc"]; pmap["nb_inplace_lshift"] = p["BinaryFunc"]; @@ -449,7 +469,7 @@ internal static Type GetPrototype(string name) return pmap[name] as Type; } - internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { Type dt; if (funcType != null) @@ -457,18 +477,15 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) else dt = GetPrototype(method.Name); - if (dt != null) + if (dt == null) { - IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); - Delegate d = Delegate.CreateDelegate(dt, method); - Thunk cb = new Thunk(d); - Marshal.StructureToPtr(cb, tmp, false); - IntPtr fp = Marshal.ReadIntPtr(tmp, 0); - Marshal.FreeHGlobal(tmp); - keepAlive.Add(d); - return fp; + return ThunkInfo.Empty; } - return IntPtr.Zero; + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + // TODO: remove keepAlive when #958 merged, let the lifecycle of ThunkInfo transfer to caller. + keepAlive.Add(info); + return info; } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -522,4 +539,22 @@ public Thunk(Delegate d) fn = d; } } + + internal class ThunkInfo + { + public readonly Delegate Target; + public readonly IntPtr Address; + + public static readonly ThunkInfo Empty = new ThunkInfo(null); + + public ThunkInfo(Delegate target) + { + if (target == null) + { + return; + } + Target = target; + Address = Marshal.GetFunctionPointerForDelegate(target); + } + } } diff --git a/src/runtime/interop27.cs b/src/runtime/interop38.cs similarity index 87% rename from src/runtime/interop27.cs rename to src/runtime/interop38.cs index 4782e9d3b..9126bca6a 100644 --- a/src/runtime/interop27.cs +++ b/src/runtime/interop38.cs @@ -1,8 +1,9 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. -#if PYTHON27 +#if PYTHON38 using System; using System.Collections; using System.Collections.Specialized; @@ -39,10 +40,10 @@ public static int magic() public static int tp_basicsize = 0; public static int tp_itemsize = 0; public static int tp_dealloc = 0; - public static int tp_print = 0; + public static int tp_vectorcall_offset = 0; public static int tp_getattr = 0; public static int tp_setattr = 0; - public static int tp_compare = 0; + public static int tp_as_async = 0; public static int tp_repr = 0; public static int tp_as_number = 0; public static int tp_as_sequence = 0; @@ -81,33 +82,34 @@ public static int magic() public static int tp_weaklist = 0; public static int tp_del = 0; public static int tp_version_tag = 0; + public static int tp_finalize = 0; + public static int tp_vectorcall = 0; + public static int tp_print = 0; + public static int am_await = 0; + public static int am_aiter = 0; + public static int am_anext = 0; public static int nb_add = 0; public static int nb_subtract = 0; public static int nb_multiply = 0; - public static int nb_divide = 0; public static int nb_remainder = 0; public static int nb_divmod = 0; public static int nb_power = 0; public static int nb_negative = 0; public static int nb_positive = 0; public static int nb_absolute = 0; - public static int nb_nonzero = 0; + public static int nb_bool = 0; public static int nb_invert = 0; public static int nb_lshift = 0; public static int nb_rshift = 0; public static int nb_and = 0; public static int nb_xor = 0; public static int nb_or = 0; - public static int nb_coerce = 0; public static int nb_int = 0; - public static int nb_long = 0; + public static int nb_reserved = 0; public static int nb_float = 0; - public static int nb_oct = 0; - public static int nb_hex = 0; public static int nb_inplace_add = 0; public static int nb_inplace_subtract = 0; public static int nb_inplace_multiply = 0; - public static int nb_inplace_divide = 0; public static int nb_inplace_remainder = 0; public static int nb_inplace_power = 0; public static int nb_inplace_lshift = 0; @@ -120,6 +122,8 @@ public static int magic() public static int nb_inplace_floor_divide = 0; public static int nb_inplace_true_divide = 0; public static int nb_index = 0; + public static int nb_matrix_multiply = 0; + public static int nb_inplace_matrix_multiply = 0; public static int mp_length = 0; public static int mp_subscript = 0; public static int mp_ass_subscript = 0; @@ -127,20 +131,18 @@ public static int magic() public static int sq_concat = 0; public static int sq_repeat = 0; public static int sq_item = 0; - public static int sq_slice = 0; + public static int was_sq_slice = 0; public static int sq_ass_item = 0; - public static int sq_ass_slice = 0; + public static int was_sq_ass_slice = 0; public static int sq_contains = 0; public static int sq_inplace_concat = 0; public static int sq_inplace_repeat = 0; - public static int bf_getreadbuffer = 0; - public static int bf_getwritebuffer = 0; - public static int bf_getsegcount = 0; - public static int bf_getcharbuffer = 0; public static int bf_getbuffer = 0; public static int bf_releasebuffer = 0; public static int name = 0; public static int ht_slots = 0; + public static int qualname = 0; + public static int ht_cached_keys = 0; /* here are optional user slots, followed by the members. */ public static int members = 0; diff --git a/src/runtime/keyvaluepairenumerableobject.cs b/src/runtime/keyvaluepairenumerableobject.cs new file mode 100644 index 000000000..c1644442c --- /dev/null +++ b/src/runtime/keyvaluepairenumerableobject.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Python.Runtime +{ + /// + /// Implements a Python type for managed KeyValuePairEnumerable (dictionaries). + /// This type is essentially the same as a ClassObject, except that it provides + /// sequence semantics to support natural dictionary usage (__contains__ and __len__) + /// from Python. + /// + internal class KeyValuePairEnumerableObject : ClassObject + { + private static Dictionary, MethodInfo> methodsByType = new Dictionary, MethodInfo>(); + private static List requiredMethods = new List { "Count", "ContainsKey" }; + + internal static bool VerifyMethodRequirements(Type type) + { + foreach (var requiredMethod in requiredMethods) + { + var method = type.GetMethod(requiredMethod); + if (method == null) + { + method = type.GetMethod($"get_{requiredMethod}"); + if (method == null) + { + return false; + } + } + + var key = Tuple.Create(type, requiredMethod); + methodsByType.Add(key, method); + } + + return true; + } + + internal KeyValuePairEnumerableObject(Type tp) : base(tp) + { + + } + + internal override bool CanSubclass() => false; + + /// + /// Implements __len__ for dictionary types. + /// + public static int mp_length(IntPtr ob) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + var key = Tuple.Create(self.GetType(), "Count"); + var methodInfo = methodsByType[key]; + + return (int)methodInfo.Invoke(self, null); + } + + /// + /// Implements __contains__ for dictionary types. + /// + public static int sq_contains(IntPtr ob, IntPtr v) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + var key = Tuple.Create(self.GetType(), "ContainsKey"); + var methodInfo = methodsByType[key]; + + var parameters = methodInfo.GetParameters(); + object arg; + if (!Converter.ToManaged(v, parameters[0].ParameterType, out arg, false)) + { + Exceptions.SetError(Exceptions.TypeError, + $"invalid parameter type for sq_contains: should be {Converter.GetTypeByAlias(v)}, found {parameters[0].ParameterType}"); + } + + return (bool)methodInfo.Invoke(self, new[] { arg }) ? 1 : 0; + } + } + + public static class KeyValuePairEnumerableObjectExtension + { + public static bool IsKeyValuePairEnumerable(this Type type) + { + var iEnumerableType = typeof(IEnumerable<>); + var keyValuePairType = typeof(KeyValuePair<,>); + + var interfaces = type.GetInterfaces(); + foreach (var i in interfaces) + { + if (i.IsGenericType && + i.GetGenericTypeDefinition() == iEnumerableType) + { + var arguments = i.GetGenericArguments(); + if (arguments.Length != 1) continue; + + var a = arguments[0]; + if (a.IsGenericType && + a.GetGenericTypeDefinition() == keyValuePairType && + a.GetGenericArguments().Length == 2) + { + return KeyValuePairEnumerableObject.VerifyMethodRequirements(type); + } + } + } + + return false; + } + } +} diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..23f5898d1 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -33,7 +33,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) { IntPtr op = tp == ob ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); + : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); if (op == IntPtr.Zero) { return null; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..5af2e1a7e 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -266,6 +266,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) return Runtime.PyFalse; } + Runtime.XIncref(args); using (var argsObj = new PyList(args)) { if (argsObj.Length() != 1) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5e800c36f..7f8e582ed 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,6 +1,9 @@ using System; using System.Collections; using System.Reflection; +using System.Text; +using System.Collections.Generic; +using System.Linq; namespace Python.Runtime { @@ -12,19 +15,18 @@ namespace Python.Runtime /// internal class MethodBinder { - public ArrayList list; - public MethodBase[] methods; + private List list; public bool init = false; public bool allow_threads = true; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MethodInformation(mi, mi.GetParameters()) }; } public int Count @@ -34,7 +36,9 @@ public int Count internal void AddMethod(MethodBase m) { - list.Add(m); + // we added a new method so we have to re sort the method list + init = false; + list.Add(new MethodInformation(m, m.GetParameters())); } /// @@ -148,22 +152,20 @@ internal static MethodInfo MatchSignatureAndParameters(MethodInfo[] mi, Type[] g return null; } - /// /// Return the array of MethodInfo for this method. The result array /// is arranged in order of precedence (done lazily to avoid doing it /// at all for methods that are never called). /// - internal MethodBase[] GetMethods() + internal List GetMethods() { if (!init) { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); init = true; } - return methods; + return list; } /// @@ -174,9 +176,10 @@ internal MethodBase[] GetMethods() /// Based from Jython `org.python.core.ReflectedArgs.precedence` /// See: https://github.com/jythontools/jython/blob/master/src/org/python/core/ReflectedArgs.java#L192 /// - internal static int GetPrecedence(MethodBase mi) + private static int GetPrecedence(MethodInformation methodInformation) { - ParameterInfo[] pi = mi.GetParameters(); + ParameterInfo[] pi = methodInformation.ParameterInfo; + var mi = methodInformation.MethodBase; int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -186,6 +189,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +210,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) @@ -278,215 +293,399 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { // loop to find match, return invoker w/ or /wo error - MethodBase[] _methods = null; - var pynargs = (int)Runtime.PyTuple_Size(args); - object arg; - var isGeneric = false; - ArrayList defaultArgList = null; - if (info != null) - { - _methods = new MethodBase[1]; - _methods.SetValue(info, 0); - } - else + + var kwargDict = new Dictionary(); + if (kw != IntPtr.Zero) { - _methods = GetMethods(); + var pynkwargs = (int) Runtime.PyDict_Size(kw); + IntPtr keylist = Runtime.PyDict_Keys(kw); + IntPtr valueList = Runtime.PyDict_Values(kw); + for (int i = 0; i < pynkwargs; ++i) + { + var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(keylist, i)); + kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i).DangerousGetAddress(); + } + + Runtime.XDecref(keylist); + Runtime.XDecref(valueList); } + + var pynargs = (int) Runtime.PyTuple_Size(args); + object arg; + var isGeneric = false; + ArrayList defaultArgList; Type clrtype; + Binding bindingUsingImplicitConversion = null; + + var methods = info == null + ? GetMethods() + : new List(1) {new MethodInformation(info, info.GetParameters())}; + // TODO: Clean up - foreach (MethodBase mi in _methods) + foreach (var methodInformation in methods) { + var mi = methodInformation.MethodBase; + var pi = methodInformation.ParameterInfo; + if (mi.IsGenericMethod) { isGeneric = true; } - ParameterInfo[] pi = mi.GetParameters(); - var clrnargs = pi.Length; - var match = false; - var arrayStart = -1; - var outs = 0; - if (pynargs == clrnargs) + bool paramsArray; + + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList)) { - match = true; + continue; } - else if (pynargs < clrnargs) + + int outs; + bool usedImplicitConversion; + + var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, + needsResolution: methods.Count > 1, + outs: out outs, + usedImplicitConversion: out usedImplicitConversion); + + if (margs == null) + { + continue; + } + + object target = null; + if (!mi.IsStatic && inst != IntPtr.Zero) { - match = true; - defaultArgList = new ArrayList(); - for (var v = pynargs; v < clrnargs; v++) + //CLRObject co = (CLRObject)ManagedType.GetManagedObject(inst); + // InvalidCastException: Unable to cast object of type + // 'Python.Runtime.ClassObject' to type 'Python.Runtime.CLRObject' + var co = ManagedType.GetManagedObject(inst) as CLRObject; + + // Sanity check: this ensures a graceful exit if someone does + // something intentionally wrong like call a non-static method + // on the class rather than on an instance of the class. + // XXX maybe better to do this before all the other rigmarole. + if (co == null) { - if (pi[v].DefaultValue == DBNull.Value) - { - match = false; - } - else - { - defaultArgList.Add(pi[v].DefaultValue); - } + return null; } + + target = co.inst; + } + + var binding = new Binding(mi, target, margs, outs); + if (usedImplicitConversion) + { + // lets just keep the first binding using implicit conversion + // this is to respect method order/precedence + if (bindingUsingImplicitConversion == null) + { + // in this case we will not return the binding yet in case there is a match + // which does not use implicit conversions, which will return directly + bindingUsingImplicitConversion = binding; + } + } + else + { + return binding; + } + } + + // if we generated a binding using implicit conversion return it + if (bindingUsingImplicitConversion != null) + { + return bindingUsingImplicitConversion; + } + + // We weren't able to find a matching method but at least one + // is a generic method and info is null. That happens when a generic + // method was not called using the [] syntax. Let's introspect the + // type of the arguments and use it to construct the correct method. + if (isGeneric && info == null && methodinfo != null) + { + Type[] types = Runtime.PythonArgsToTypeArray(args, true); + MethodInfo mi = MatchParameters(methodinfo, types); + return Bind(inst, args, kw, mi, null); + } + return null; + } + + static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out bool isNewReference) + { + isNewReference = false; + IntPtr op; + // for a params method, we may have a sequence or single/multiple items + // here we look to see if the item at the paramIndex is there or not + // and then if it is a sequence itself. + if ((pyArgCount - arrayStart) == 1) + { + // we only have one argument left, so we need to check it + // to see if it is a sequence or a single item + IntPtr item = Runtime.PyTuple_GetItem(args, arrayStart); + if (!Runtime.PyString_Check(item) && Runtime.PySequence_Check(item)) + { + // it's a sequence (and not a string), so we use it as the op + op = item; } - else if (pynargs > clrnargs && clrnargs > 0 && - Attribute.IsDefined(pi[clrnargs - 1], typeof(ParamArrayAttribute))) + else { - // This is a `foo(params object[] bar)` style method - match = true; - arrayStart = clrnargs - 1; + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); } + } + else + { + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); + } + return op; + } + + /// + /// Attempts to convert Python positional argument tuple and keyword argument table + /// into an array of managed objects, that can be passed to a method. + /// + /// Information about expected parameters + /// true, if the last parameter is a params array. + /// A pointer to the Python argument tuple + /// Number of arguments, passed by Python + /// Dictionary of keyword argument name to python object pointer + /// A list of default values for omitted parameters + /// true, if overloading resolution is required + /// Returns number of output parameters + /// An array of .NET arguments, that can be passed to a method. + static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, + IntPtr args, int pyArgCount, + Dictionary kwargDict, + ArrayList defaultArgList, + bool needsResolution, + out int outs, + out bool usedImplicitConversion) + { + outs = 0; + usedImplicitConversion = false; + var margs = new object[pi.Length]; + int arrayStart = paramsArray ? pi.Length - 1 : -1; + + for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) + { + var parameter = pi[paramIndex]; + bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + bool isNewReference = false; - if (match) + if (paramIndex >= pyArgCount && !(hasNamedParam || (paramsArray && paramIndex == arrayStart))) { - var margs = new object[clrnargs]; + if (defaultArgList != null) + { + margs[paramIndex] = defaultArgList[paramIndex - pyArgCount]; + } + + continue; + } - for (int n = 0; n < clrnargs; n++) + IntPtr op; + if (hasNamedParam) + { + op = kwargDict[parameter.Name]; + } + else + { + if(arrayStart == paramIndex) { - IntPtr op; - if (n < pynargs) - { - if (arrayStart == n) - { - // map remaining Python arguments to a tuple since - // the managed function accepts it - hopefully :] - op = Runtime.PyTuple_GetSlice(args, arrayStart, pynargs); - } - else - { - op = Runtime.PyTuple_GetItem(args, n); - } + op = HandleParamsArray(args, arrayStart, pyArgCount, out isNewReference); + } + else + { + op = Runtime.PyTuple_GetItem(args, paramIndex); + } + } - // this logic below handles cases when multiple overloading methods - // are ambiguous, hence comparison between Python and CLR types - // is necessary - clrtype = null; - IntPtr pyoptype; - if (_methods.Length > 1) - { - pyoptype = IntPtr.Zero; - pyoptype = Runtime.PyObject_Type(op); - Exceptions.Clear(); - if (pyoptype != IntPtr.Zero) - { - clrtype = Converter.GetTypeByAlias(pyoptype); - } - Runtime.XDecref(pyoptype); - } + bool isOut; + if (!TryConvertArgument(op, parameter.ParameterType, needsResolution, out margs[paramIndex], out isOut, out usedImplicitConversion)) + { + return null; + } + if (isNewReference) + { + // TODO: is this a bug? Should this happen even if the conversion fails? + // GetSlice() creates a new reference but GetItem() + // returns only a borrow reference. + Runtime.XDecref(op); + } - if (clrtype != null) - { - var typematch = false; - if ((pi[n].ParameterType != typeof(object)) && (pi[n].ParameterType != clrtype)) - { - IntPtr pytype = Converter.GetPythonTypeByAlias(pi[n].ParameterType); - pyoptype = Runtime.PyObject_Type(op); - Exceptions.Clear(); - if (pyoptype != IntPtr.Zero) - { - if (pytype != pyoptype) - { - typematch = false; - } - else - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - if (!typematch) - { - // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); - TypeCode paramtypecode = Type.GetTypeCode(clrtype); - if (argtypecode == paramtypecode) - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - Runtime.XDecref(pyoptype); - if (!typematch) - { - margs = null; - break; - } - } - else - { - typematch = true; - clrtype = pi[n].ParameterType; - } - } - else - { - clrtype = pi[n].ParameterType; - } + if (parameter.IsOut || isOut) + { + outs++; + } + } - if (pi[n].IsOut || clrtype.IsByRef) - { - outs++; - } + return margs; + } - if (!Converter.ToManaged(op, clrtype, out arg, false)) - { - Exceptions.Clear(); - margs = null; - break; - } - if (arrayStart == n) - { - // GetSlice() creates a new reference but GetItem() - // returns only a borrow reference. - Runtime.XDecref(op); - } - margs[n] = arg; + static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, + out object arg, out bool isOut, out bool usedImplicitConversion) + { + arg = null; + isOut = false; + usedImplicitConversion = false; + var clrtype = TryComputeClrArgumentType(parameterType, op, needsResolution: needsResolution, out usedImplicitConversion); + if (clrtype == null) + { + return false; + } + + if (!Converter.ToManaged(op, clrtype, out arg, false)) + { + Exceptions.Clear(); + return false; + } + + isOut = clrtype.IsByRef; + return true; + } + + static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool needsResolution, out bool usedImplicitConversion) + { + // this logic below handles cases when multiple overloading methods + // are ambiguous, hence comparison between Python and CLR types + // is necessary + usedImplicitConversion = false; + Type clrtype = null; + IntPtr pyoptype; + if (needsResolution) + { + // HACK: each overload should be weighted in some way instead + pyoptype = Runtime.PyObject_Type(argument); + Exceptions.Clear(); + if (pyoptype != IntPtr.Zero) + { + clrtype = Converter.GetTypeByAlias(pyoptype); + } + Runtime.XDecref(pyoptype); + } + + if (clrtype != null) + { + var typematch = false; + if ((parameterType != typeof(object)) && (parameterType != clrtype)) + { + IntPtr pytype = Converter.GetPythonTypeByAlias(parameterType); + pyoptype = Runtime.PyObject_Type(argument); + Exceptions.Clear(); + if (pyoptype != IntPtr.Zero) + { + if (pytype != pyoptype) + { + typematch = false; } else { - if (defaultArgList != null) - { - margs[n] = defaultArgList[n - pynargs]; - } + typematch = true; + clrtype = parameterType; } } - - if (margs == null) + if (!typematch) { - continue; - } - - object target = null; - if (!mi.IsStatic && inst != IntPtr.Zero) - { - //CLRObject co = (CLRObject)ManagedType.GetManagedObject(inst); - // InvalidCastException: Unable to cast object of type - // 'Python.Runtime.ClassObject' to type 'Python.Runtime.CLRObject' - var co = ManagedType.GetManagedObject(inst) as CLRObject; - - // Sanity check: this ensures a graceful exit if someone does - // something intentionally wrong like call a non-static method - // on the class rather than on an instance of the class. - // XXX maybe better to do this before all the other rigmarole. - if (co == null) + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(parameterType); + if (underlyingType == null) + { + underlyingType = parameterType; + } + // this takes care of enum values + TypeCode argtypecode = Type.GetTypeCode(underlyingType); + TypeCode paramtypecode = Type.GetTypeCode(clrtype); + if (argtypecode == paramtypecode) { - return null; + typematch = true; + clrtype = parameterType; } - target = co.inst; + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + object arg; + clrtype = parameterType; + typematch = Converter.ToManaged(argument, clrtype, out arg, false); + } + // this takes care of implicit conversions + var opImplicit = parameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + usedImplicitConversion = typematch = opImplicit.ReturnType == parameterType; + clrtype = parameterType; + } + } + Runtime.XDecref(pyoptype); + if (!typematch) + { + return null; } + } + else + { + typematch = true; + clrtype = parameterType; + } + } + else + { + clrtype = parameterType; + } + + return clrtype; + } - return new Binding(mi, target, margs, outs); + static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, + Dictionary kwargDict, + out bool paramsArray, + out ArrayList defaultArgList) + { + defaultArgList = null; + var match = false; + paramsArray = parameters.Length > 0 ? Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute)) : false; + + if (positionalArgumentCount == parameters.Length) + { + match = true; + } + else if (positionalArgumentCount < parameters.Length) + { + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword argument or a default parameter + match = true; + defaultArgList = new ArrayList(); + for (var v = positionalArgumentCount; v < parameters.Length; v++) + { + if (kwargDict.ContainsKey(parameters[v].Name)) + { + // we have a keyword argument for this parameter, + // no need to check for a default parameter, but put a null + // placeholder in defaultArgList + defaultArgList.Add(null); + } + else if (parameters[v].IsOptional) + { + // IsOptional will be true if the parameter has a default value, + // or if the parameter has the [Optional] attribute specified. + // The GetDefaultValue() extension method will return the value + // to be passed in as the parameter value + defaultArgList.Add(parameters[v].GetDefaultValue()); + } + else if(!paramsArray) + { + match = false; + } } } - // We weren't able to find a matching method but at least one - // is a generic method and info is null. That happens when a generic - // method was not called using the [] syntax. Let's introspect the - // type of the arguments and use it to construct the correct method. - if (isGeneric && info == null && methodinfo != null) + else if (positionalArgumentCount > parameters.Length && parameters.Length > 0 && + Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute))) { - Type[] types = Runtime.PythonArgsToTypeArray(args, true); - MethodInfo mi = MatchParameters(methodinfo, types); - return Bind(inst, args, kw, mi, null); + // This is a `foo(params object[] bar)` style method + match = true; + paramsArray = true; } - return null; + + return match; } internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) @@ -499,6 +698,40 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Invoke(inst, args, kw, info, null); } + protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) + { + long argCount = Runtime.PyTuple_Size(args); + to.Append("("); + for (long argIndex = 0; argIndex < argCount; argIndex++) + { + var arg = Runtime.PyTuple_GetItem(args, argIndex); + if (arg != IntPtr.Zero) + { + var type = Runtime.PyObject_Type(arg); + if (type != IntPtr.Zero) + { + try + { + var description = Runtime.PyObject_Unicode(type); + if (description != IntPtr.Zero) + { + to.Append(Runtime.GetManagedString(description)); + Runtime.XDecref(description); + } + } + finally + { + Runtime.XDecref(type); + } + } + } + + if (argIndex + 1 < argCount) + to.Append(", "); + } + to.Append(')'); + } + internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { Binding binding = Bind(inst, args, kw, info, methodinfo); @@ -507,12 +740,15 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding == null) { - var value = "No method matches given arguments"; + var value = new StringBuilder("No method matches given arguments"); if (methodinfo != null && methodinfo.Length > 0) { - value += $" for {methodinfo[0].Name}"; + value.Append($" for {methodinfo[0].Name}"); } - Exceptions.SetError(Exceptions.TypeError, value); + + value.Append(": "); + AppendArgumentTypes(to: value, args); + Exceptions.SetError(Exceptions.TypeError, value.ToString()); return IntPtr.Zero; } @@ -591,44 +827,52 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Converter.ToPython(result, mi.ReturnType); } - } - /// - /// Utility class to sort method info by parameter type precedence. - /// - internal class MethodSorter : IComparer - { - int IComparer.Compare(object m1, object m2) + /// + /// Utility class to store the information about a + /// + internal class MethodInformation { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; + public MethodBase MethodBase { get; } + public ParameterInfo[] ParameterInfo { get; } - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; + public MethodInformation(MethodBase methodBase, ParameterInfo[] parameterInfo) + { + MethodBase = methodBase; + ParameterInfo = parameterInfo; } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); - if (p1 < p2) + public override string ToString() { - return -1; + return MethodBase.ToString(); } - if (p1 > p2) + } + + /// + /// Utility class to sort method info by parameter type precedence. + /// + private class MethodSorter : IComparer + { + public int Compare(MethodInformation x, MethodInformation y) { - return 1; + int p1 = GetPrecedence(x); + int p2 = GetPrecedence(y); + if (p1 < p2) + { + return -1; + } + + if (p1 > p2) + { + return 1; + } + + return 0; } - return 0; } } - /// /// A Binding is a utility instance that bundles together a MethodInfo /// representing a method to call, a (possibly null) target instance for @@ -649,4 +893,33 @@ internal Binding(MethodBase info, object inst, object[] args, int outs) this.outs = outs; } } + + + static internal class ParameterInfoExtensions + { + public static object GetDefaultValue(this ParameterInfo parameterInfo) + { + // parameterInfo.HasDefaultValue is preferable but doesn't exist in .NET 4.0 + bool hasDefaultValue = (parameterInfo.Attributes & ParameterAttributes.HasDefault) == + ParameterAttributes.HasDefault; + + if (hasDefaultValue) + { + return parameterInfo.DefaultValue; + } + else + { + // [OptionalAttribute] was specified for the parameter. + // See https://stackoverflow.com/questions/3416216/optionalattribute-parameters-default-value + // for rules on determining the value to pass to the parameter + var type = parameterInfo.ParameterType; + if (type == typeof(object)) + return Type.Missing; + else if (type.IsValueType) + return Activator.CreateInstance(type); + else + return null; + } + } + } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..2e67787d2 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -68,14 +68,14 @@ internal IntPtr GetDocString() } var str = ""; Type marker = typeof(DocStringAttribute); - MethodBase[] methods = binder.GetMethods(); - foreach (MethodBase method in methods) + var methods = binder.GetMethods(); + foreach (var method in methods) { if (str.Length > 0) { str += Environment.NewLine; } - var attrs = (Attribute[])method.GetCustomAttributes(marker, false); + var attrs = (Attribute[])method.MethodBase.GetCustomAttributes(marker, false); if (attrs.Length == 0) { str += method.ToString(); diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index 2f3ce3ef2..bc7500dab 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,18 +12,21 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private bool _disposed = false; + + private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) { // Turn the managed method into a function pointer - IntPtr fp = Interop.GetThunk(type.GetMethod(name), funcType); + _thunk = Interop.GetThunk(type.GetMethod(name), funcType); // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); - TypeManager.WriteMethodDef(mdef, name, fp, 0x0003); + TypeManager.WriteMethodDef(mdef, name, _thunk.Address, 0x0003); ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } @@ -31,5 +34,21 @@ public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } + + public void Release() + { + if (_disposed) + { + return; + } + _disposed = true; + bool freeDef = Runtime.Refcount(ptr) == 1; + Runtime.XDecref(ptr); + if (freeDef && mdef != IntPtr.Zero) + { + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; + } + } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8af722d29..3d4127304 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -53,7 +54,7 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); InitializeModuleMembers(); } @@ -67,9 +68,8 @@ public ModuleObject(string name) /// public ManagedType GetAttribute(string name, bool guess) { - ManagedType cached = null; - cache.TryGetValue(name, out cached); - if (cached != null) + ManagedType cached; + if (cache.TryGetValue(name, out cached)) { return cached; } @@ -78,16 +78,6 @@ public ManagedType GetAttribute(string name, bool guess) ClassBase c; Type type; - //if (AssemblyManager.IsValidNamespace(name)) - //{ - // IntPtr py_mod_name = Runtime.PyString_FromString(name); - // IntPtr modules = Runtime.PyImport_GetModuleDict(); - // IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name); - // if (module != IntPtr.Zero) - // return (ManagedType)this; - // return null; - //} - string qname = _namespace == string.Empty ? name : _namespace + "." + name; @@ -105,13 +95,9 @@ public ManagedType GetAttribute(string name, bool guess) // Look for a type in the current namespace. Note that this // includes types, delegates, enums, interfaces and structs. // Only public namespace members are exposed to Python. - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -131,13 +117,9 @@ public ManagedType GetAttribute(string name, bool guess) return m; } - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -186,21 +168,9 @@ private void StoreAttribute(string name, ManagedType ob) /// public void LoadNames() { - ManagedType m = null; - foreach (string name in AssemblyManager.GetNames(_namespace)) + foreach (var name in AssemblyManager.GetNames(_namespace)) { - cache.TryGetValue(name, out m); - if (m != null) - { - continue; - } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); - // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) - { - continue; - } - GetAttribute(name, true); + var attr = GetAttribute(name, true); } } @@ -280,7 +250,18 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } - ManagedType attr = self.GetAttribute(name, true); + ManagedType attr = null; + + try + { + attr = self.GetAttribute(name, true); + } + catch (Exception e) + { + Exceptions.SetError(e); + return IntPtr.Zero; + } + if (attr == null) { @@ -316,6 +297,11 @@ internal class CLRModule : ModuleObject internal static bool _SuppressDocs = false; internal static bool _SuppressOverloads = false; + static CLRModule() + { + Reset(); + } + public CLRModule() : base("clr") { _namespace = string.Empty; @@ -413,14 +399,6 @@ public static Assembly AddReference(string name) { assembly = AssemblyManager.LoadAssemblyFullPath(name); } - if (System.IO.File.Exists(name)) - { - var zone = System.Security.Policy.Zone.CreateFromUrl(name); - if (zone.SecurityZone != System.Security.SecurityZone.MyComputer) - { - throw new Exception($"File is blocked (NTFS Security)"); - } - } if (assembly == null) { throw new FileNotFoundException($"Unable to find assembly '{name}'."); diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index b5bf25dd7..4a7bf05c8 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -32,19 +32,21 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - ((Void_1_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Void_1_Delegate)))(a1); + var d = Marshal.GetDelegateForFunctionPointer(fp); + d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = (Interop.TernaryFunc)Marshal.GetDelegateForFunctionPointer(fp, typeof(Interop.TernaryFunc)); + var d = Marshal.GetDelegateForFunctionPointer(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - return ((Int_3_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Int_3_Delegate)))(a1, a2, a3); + var d = Marshal.GetDelegateForFunctionPointer(fp); + return d(a1, a2, a3); } #else private static AssemblyBuilder aBuilder; diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs new file mode 100644 index 000000000..a6d88cd19 --- /dev/null +++ b/src/runtime/platform/LibraryLoader.cs @@ -0,0 +1,208 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibraryLoader + { + IntPtr Load(string dllToLoad); + + IntPtr GetFunction(IntPtr hModule, string procedureName); + + void Free(IntPtr hModule); + } + + static class LibraryLoader + { + public static ILibraryLoader Get(OperatingSystemType os) + { + switch (os) + { + case OperatingSystemType.Windows: + return new WindowsLoader(); + case OperatingSystemType.Darwin: + return new DarwinLoader(); + case OperatingSystemType.Linux: + return new LinuxLoader(); + default: + throw new PlatformNotSupportedException($"This operating system ({os}) is not supported"); + } + } + } + + class LinuxLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x100; + private static IntPtr RTLD_DEFAULT = IntPtr.Zero; + private const string NativeDll = "libdl.so"; + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.so"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class DarwinLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x8; + private const string NativeDll = "/usr/lib/libSystem.dylib"; + private static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.dylib"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class WindowsLoader : ILibraryLoader + { + private const string NativeDll = "kernel32.dll"; + + + public IntPtr Load(string dllToLoad) + { + var res = WindowsLoader.LoadLibrary(dllToLoad); + if (res == IntPtr.Zero) + throw new DllNotFoundException($"Could not load {dllToLoad}", new Win32Exception()); + return res; + } + + public IntPtr GetFunction(IntPtr hModule, string procedureName) + { + var res = WindowsLoader.GetProcAddress(hModule, procedureName); + if (res == IntPtr.Zero) + throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); + return res; + } + + public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr LoadLibrary(string dllToLoad); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + [DllImport(NativeDll)] + static extern bool FreeLibrary(IntPtr hModule); + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs new file mode 100644 index 000000000..62be0e421 --- /dev/null +++ b/src/runtime/platform/Types.cs @@ -0,0 +1,23 @@ +namespace Python.Runtime.Platform +{ + public enum MachineType + { + i386, + x86_64, + armv7l, + armv8, + aarch64, + Other + }; + + /// + /// Operating system type as reported by Python. + /// + public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } +} diff --git a/src/runtime/polyfill/ReflectionPolifills.cs b/src/runtime/polyfill/ReflectionPolifills.cs index a7e9c879a..b9ce78d63 100644 --- a/src/runtime/polyfill/ReflectionPolifills.cs +++ b/src/runtime/polyfill/ReflectionPolifills.cs @@ -1,12 +1,14 @@ using System; +using System.Linq; using System.Reflection; using System.Reflection.Emit; namespace Python.Runtime { -#if NETSTANDARD + [Obsolete("This API is for internal use only")] public static class ReflectionPolifills { +#if NETSTANDARD public static AssemblyBuilder DefineDynamicAssembly(this AppDomain appDomain, AssemblyName assemblyName, AssemblyBuilderAccess assemblyBuilderAccess) { return AssemblyBuilder.DefineDynamicAssembly(assemblyName, assemblyBuilderAccess); @@ -16,6 +18,19 @@ public static Type CreateType(this TypeBuilder typeBuilder) { return typeBuilder.GetTypeInfo().GetType(); } - } #endif + public static T GetCustomAttribute(this Type type) where T: Attribute + { + return type.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + + public static T GetCustomAttribute(this Assembly assembly) where T: Attribute + { + return assembly.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + } } diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..cc6125157 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -1,4 +1,5 @@ using System; +using System.Linq.Expressions; using System.Reflection; using System.Security.Permissions; @@ -12,6 +13,10 @@ internal class PropertyObject : ExtensionType private PropertyInfo info; private MethodInfo getter; private MethodInfo setter; + private bool getterCacheFailed; + private bool setterCacheFailed; + private Func getterCache; + private Action setterCache; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -67,7 +72,21 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); + if (self.getterCache == null && !self.getterCacheFailed) + { + // if the getter is not public 'GetGetMethod' will not find it + // but calling 'GetValue' will work, so for backwards compatibility + // we will use it instead + self.getterCache = BuildGetter(self.info); + if (self.getterCache == null) + { + self.getterCacheFailed = true; + } + } + + result = self.getterCacheFailed ? + self.info.GetValue(co.inst, null) + : self.getterCache(co.inst); return Converter.ToPython(result, self.info.PropertyType); } catch (Exception e) @@ -81,7 +100,6 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } } - /// /// Descriptor __set__ implementation. This method sets the value of /// a property based on the given Python value. The Python value must @@ -132,7 +150,27 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + + if (self.setterCache == null && !self.setterCacheFailed) + { + // if the setter is not public 'GetSetMethod' will not find it + // but calling 'SetValue' will work, so for backwards compatibility + // we will use it instead + self.setterCache = BuildSetter(self.info); + if (self.setterCache == null) + { + self.setterCacheFailed = true; + } + } + + if (self.setterCacheFailed) + { + self.info.SetValue(co.inst, newval, null); + } + else + { + self.setterCache(co.inst, newval); + } } else { @@ -160,5 +198,54 @@ public static IntPtr tp_repr(IntPtr ob) var self = (PropertyObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + private static Func BuildGetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetGetMethod(); + if (methodInfo == null) + { + // if the getter is not public 'GetGetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var expressionCall = Expression.Call(instance, methodInfo); + + return Expression.Lambda>( + Expression.Convert(expressionCall, typeof(object)), + obj).Compile(); + } + + private static Action BuildSetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetSetMethod(); + if (methodInfo == null) + { + // if the setter is not public 'GetSetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var parameters = methodInfo.GetParameters(); + if (parameters.Length != 1) + { + return null; + } + var value = Expression.Parameter(typeof(object)); + var argument = Expression.Convert(value, parameters[0].ParameterType); + + var expressionCall = Expression.Call(instance, methodInfo, argument); + + return Expression.Lambda>( + expressionCall, + obj, + value).Compile(); + } } } diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 7237d1990..7ff7a83c8 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -139,12 +139,20 @@ public PyObject Values() /// public PyObject Items() { - IntPtr items = Runtime.PyDict_Items(obj); - if (items == IntPtr.Zero) + var items = Runtime.PyDict_Items(this.obj); + try { - throw new PythonException(); + if (items.IsNull()) + { + throw new PythonException(); + } + + return items.MoveToPyObject(); + } + finally + { + items.Dispose(); } - return new PyObject(items); } diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f6911d9d7..217cf7e20 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -62,7 +62,7 @@ public PyInt(int value) /// Creates a new Python int from a uint32 value. /// [CLSCompliant(false)] - public PyInt(uint value) : base(IntPtr.Zero) + public PyInt(uint value) { obj = Runtime.PyInt_FromInt64(value); Runtime.CheckExceptionOccurred(); @@ -75,7 +75,7 @@ public PyInt(uint value) : base(IntPtr.Zero) /// /// Creates a new Python int from an int64 value. /// - public PyInt(long value) : base(IntPtr.Zero) + public PyInt(long value) { obj = Runtime.PyInt_FromInt64(value); Runtime.CheckExceptionOccurred(); @@ -89,7 +89,7 @@ public PyInt(long value) : base(IntPtr.Zero) /// Creates a new Python int from a uint64 value. /// [CLSCompliant(false)] - public PyInt(ulong value) : base(IntPtr.Zero) + public PyInt(ulong value) { obj = Runtime.PyInt_FromInt64((long)value); Runtime.CheckExceptionOccurred(); diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index b22d9d51f..347cc3000 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -120,7 +120,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(obj, item.obj); + int r = Runtime.PyList_Append(this.Reference, item.obj); if (r < 0) { throw new PythonException(); @@ -135,7 +135,7 @@ public void Append(PyObject item) /// public void Insert(int index, PyObject item) { - int r = Runtime.PyList_Insert(obj, index, item.obj); + int r = Runtime.PyList_Insert(this.Reference, index, item.obj); if (r < 0) { throw new PythonException(); @@ -151,7 +151,7 @@ public void Insert(int index, PyObject item) /// public void Reverse() { - int r = Runtime.PyList_Reverse(obj); + int r = Runtime.PyList_Reverse(this.Reference); if (r < 0) { throw new PythonException(); @@ -167,7 +167,7 @@ public void Reverse() /// public void Sort() { - int r = Runtime.PyList_Sort(obj); + int r = Runtime.PyList_Sort(this.Reference); if (r < 0) { throw new PythonException(); diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b8c71efa..85787a428 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,11 +1,18 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; +using System.Linq; using System.Linq.Expressions; namespace Python.Runtime { + public interface IPyDisposable : IDisposable + { + IntPtr[] GetTrackedHandles(); + } + /// /// Represents a generic Python object. The methods of this class are /// generally equivalent to the Python "abstract object API". See @@ -13,10 +20,18 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IDisposable + public class PyObject : DynamicObject, IEnumerable, IPyDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; - private bool disposed = false; + + internal BorrowedReference Reference => new BorrowedReference(obj); /// /// PyObject Constructor @@ -29,26 +44,51 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable /// public PyObject(IntPtr ptr) { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); + obj = ptr; + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif + } + + /// + /// Creates new pointing to the same object as + /// the . Increments refcount, allowing + /// to have ownership over its own reference. + /// + internal PyObject(BorrowedReference reference) + { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + obj = Runtime.SelfIncRef(reference.DangerousGetAddress()); + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Protected default constructor to allow subclasses to manage // initialization in different ways as appropriate. - + [Obsolete("Please, always use PyObject(*Reference)")] protected PyObject() { + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (obj == IntPtr.Zero) + { + return; + } + Finalizer.Instance.AddFinalizedObject(this); } @@ -66,7 +106,8 @@ public IntPtr Handle /// - /// FromManagedObject Method + /// Gets raw Python proxy for this object (bypasses all conversions, + /// except null <==> None) /// /// /// Given an arbitrary managed object, return a Python instance that @@ -101,7 +142,7 @@ public object AsManagedObject(Type t) } return result; } - + /// /// As Method /// @@ -137,17 +178,41 @@ public T As() /// protected virtual void Dispose(bool disposing) { - if (!disposed) + if (this.obj == IntPtr.Zero) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + return; + } + + if (Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be initialized"); + + if (!Runtime.IsFinalizing) + { + long refcount = Runtime.Refcount(this.obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + + if (refcount == 1) + { + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + + try + { + Runtime.XDecref(this.obj); + Runtime.CheckExceptionOccurred(); + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType, errVal, traceback); + } + } + else { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(obj); - obj = IntPtr.Zero; - PythonEngine.ReleaseLock(gs); + Runtime.XDecref(this.obj); } - disposed = true; } + this.obj = IntPtr.Zero; } public void Dispose() @@ -156,6 +221,10 @@ public void Dispose() GC.SuppressFinalize(this); } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } /// /// GetPythonType Method @@ -170,7 +239,6 @@ public PyObject GetPythonType() return new PyObject(tp); } - /// /// TypeCheck Method /// @@ -180,6 +248,8 @@ public PyObject GetPythonType() /// public bool TypeCheck(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + return Runtime.PyObject_TypeCheck(obj, typeOrClass.obj); } @@ -192,6 +262,8 @@ public bool TypeCheck(PyObject typeOrClass) /// public bool HasAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttrString(obj, name) != 0; } @@ -205,6 +277,8 @@ public bool HasAttr(string name) /// public bool HasAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttr(obj, name.obj) != 0; } @@ -218,6 +292,8 @@ public bool HasAttr(PyObject name) /// public PyObject GetAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -228,7 +304,7 @@ public PyObject GetAttr(string name) /// - /// GetAttr Method + /// GetAttr Method. Returns fallback value if getting attribute fails for any reason. /// /// /// Returns the named attribute of the Python object, or the given @@ -236,6 +312,8 @@ public PyObject GetAttr(string name) /// public PyObject GetAttr(string name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -256,6 +334,8 @@ public PyObject GetAttr(string name, PyObject _default) /// public PyObject GetAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -275,6 +355,8 @@ public PyObject GetAttr(PyObject name) /// public PyObject GetAttr(PyObject name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -294,6 +376,9 @@ public PyObject GetAttr(PyObject name, PyObject _default) /// public void SetAttr(string name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); if (r < 0) { @@ -312,6 +397,9 @@ public void SetAttr(string name, PyObject value) /// public void SetAttr(PyObject name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); if (r < 0) { @@ -329,6 +417,8 @@ public void SetAttr(PyObject name, PyObject value) /// public void DelAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttrString(obj, name, IntPtr.Zero); if (r < 0) { @@ -347,6 +437,8 @@ public void DelAttr(string name) /// public void DelAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, IntPtr.Zero); if (r < 0) { @@ -365,6 +457,8 @@ public void DelAttr(PyObject name) /// public virtual PyObject GetItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + IntPtr op = Runtime.PyObject_GetItem(obj, key.obj); if (op == IntPtr.Zero) { @@ -384,6 +478,8 @@ public virtual PyObject GetItem(PyObject key) /// public virtual PyObject GetItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { return GetItem(pyKey); @@ -418,6 +514,9 @@ public virtual PyObject GetItem(int index) /// public virtual void SetItem(PyObject key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); if (r < 0) { @@ -436,6 +535,9 @@ public virtual void SetItem(PyObject key, PyObject value) /// public virtual void SetItem(string key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyKey = new PyString(key)) { SetItem(pyKey, value); @@ -453,6 +555,8 @@ public virtual void SetItem(string key, PyObject value) /// public virtual void SetItem(int index, PyObject value) { + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyindex = new PyInt(index)) { SetItem(pyindex, value); @@ -470,6 +574,8 @@ public virtual void SetItem(int index, PyObject value) /// public virtual void DelItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + int r = Runtime.PyObject_DelItem(obj, key.obj); if (r < 0) { @@ -488,6 +594,8 @@ public virtual void DelItem(PyObject key) /// public virtual void DelItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { DelItem(pyKey); @@ -610,10 +718,13 @@ public IEnumerator GetEnumerator() /// /// /// Invoke the callable object with the given arguments, passed as a - /// PyObject[]. A PythonException is raised if the invokation fails. + /// PyObject[]. A PythonException is raised if the invocation fails. /// public PyObject Invoke(params PyObject[] args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); IntPtr r = Runtime.PyObject_Call(obj, t.obj, IntPtr.Zero); t.Dispose(); @@ -630,10 +741,12 @@ public PyObject Invoke(params PyObject[] args) /// /// /// Invoke the callable object with the given arguments, passed as a - /// Python tuple. A PythonException is raised if the invokation fails. + /// Python tuple. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + IntPtr r = Runtime.PyObject_Call(obj, args.obj, IntPtr.Zero); if (r == IntPtr.Zero) { @@ -648,12 +761,15 @@ public PyObject Invoke(PyTuple args) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyObject[] args, PyDict kw) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); - IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw != null ? kw.obj : IntPtr.Zero); + IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw?.obj ?? IntPtr.Zero); t.Dispose(); if (r == IntPtr.Zero) { @@ -668,11 +784,13 @@ public PyObject Invoke(PyObject[] args, PyDict kw) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args, PyDict kw) { - IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw != null ? kw.obj : IntPtr.Zero); + if (args == null) throw new ArgumentNullException(nameof(args)); + + IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw?.obj ?? IntPtr.Zero); if (r == IntPtr.Zero) { throw new PythonException(); @@ -686,10 +804,14 @@ public PyObject Invoke(PyTuple args, PyDict kw) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, params PyObject[] args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -702,10 +824,51 @@ public PyObject InvokeMethod(string name, params PyObject[] args) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, params PyObject[] args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, PyTuple args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -719,10 +882,14 @@ public PyObject InvokeMethod(string name, PyTuple args) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -736,10 +903,13 @@ public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -756,6 +926,8 @@ public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) /// public bool IsInstance(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsInstance(obj, typeOrClass.obj); if (r < 0) { @@ -775,6 +947,8 @@ public bool IsInstance(PyObject typeOrClass) /// public bool IsSubclass(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsSubclass(obj, typeOrClass.obj); if (r < 0) { @@ -823,6 +997,10 @@ public bool IsTrue() return Runtime.PyObject_IsTrue(obj) != 0; } + /// + /// Return true if the object is None + /// + public bool IsNone() => CheckNone(this) == null; /// /// Dir Method @@ -1121,10 +1299,10 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceMultiply(this.obj, ((PyObject)arg).obj); break; case ExpressionType.Divide: - res = Runtime.PyNumber_Divide(this.obj, ((PyObject)arg).obj); + res = Runtime.PyNumber_TrueDivide(this.obj, ((PyObject)arg).obj); break; case ExpressionType.DivideAssign: - res = Runtime.PyNumber_InPlaceDivide(this.obj, ((PyObject)arg).obj); + res = Runtime.PyNumber_InPlaceTrueDivide(this.obj, ((PyObject)arg).obj); break; case ExpressionType.And: res = Runtime.PyNumber_And(this.obj, ((PyObject)arg).obj); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8e6957855..8738824f5 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,7 +22,7 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : DynamicObject, IPyDisposable { public readonly string Name; @@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable internal readonly IntPtr variables; private bool _isDisposed; + private bool _finalized = false; /// /// The Manager this scope associated with. @@ -277,11 +278,19 @@ public PyObject Eval(string code, PyDict locals = null) Check(); IntPtr _locals = locals == null ? variables : locals.obj; var flag = (IntPtr)Runtime.Py_eval_input; - IntPtr ptr = Runtime.PyRun_String( + + NewReference reference = Runtime.PyRun_String( code, flag, variables, _locals ); - Runtime.CheckExceptionOccurred(); - return new PyObject(ptr); + try + { + Runtime.CheckExceptionOccurred(); + return reference.MoveToPyObject(); + } + finally + { + reference.Dispose(); + } } /// @@ -315,15 +324,22 @@ public void Exec(string code, PyDict locals = null) private void Exec(string code, IntPtr _globals, IntPtr _locals) { var flag = (IntPtr)Runtime.Py_file_input; - IntPtr ptr = Runtime.PyRun_String( + NewReference reference = Runtime.PyRun_String( code, flag, _globals, _locals ); - Runtime.CheckExceptionOccurred(); - if (ptr != Runtime.PyNone) + + try { - throw new PythonException(); + Runtime.CheckExceptionOccurred(); + if (!reference.IsNone()) + { + throw new PythonException(); + } + } + finally + { + reference.Dispose(); } - Runtime.XDecref(ptr); } /// @@ -525,13 +541,19 @@ public void Dispose() this.OnDispose?.Invoke(this); } - ~PyScope() + public IntPtr[] GetTrackedHandles() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + return new IntPtr[] { obj }; + } - Dispose(); + ~PyScope() + { + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..4f77bec1d 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -80,6 +80,7 @@ public static string PythonHome } set { + // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); Runtime.Py_SetPythonHome(_pythonHome); @@ -95,10 +96,6 @@ public static string PythonPath } set { - if (Runtime.IsPython2) - { - throw new NotSupportedException("Set PythonPath not supported on Python 2"); - } Marshal.FreeHGlobal(_pythonPath); _pythonPath = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); Runtime.Py_SetPath(_pythonPath); @@ -130,6 +127,16 @@ public static string Compiler get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } } + /// + /// Set the NoSiteFlag to disable loading the site module. + /// Must be called before Initialize. + /// https://docs.python.org/3/c-api/init.html#c.Py_NoSiteFlag + /// + public static void SetNoSiteFlag() + { + Runtime.SetNoSiteFlag(); + } + public static int RunSimpleString(string code) { return Runtime.PyRun_SimpleString(code); @@ -244,11 +251,7 @@ static void OnDomainUnload(object _, EventArgs __) /// CPython interpreter process - this bootstraps the managed runtime /// when it is imported by the CLR extension module. /// -#if PYTHON3 public static IntPtr InitExt() -#elif PYTHON2 - public static void InitExt() -#endif { try { @@ -288,14 +291,10 @@ public static void InitExt() catch (PythonException e) { e.Restore(); -#if PYTHON3 return IntPtr.Zero; -#endif } -#if PYTHON3 return Python.Runtime.ImportHook.GetCLRModule(); -#endif } /// @@ -311,13 +310,15 @@ public static void Shutdown() if (initialized) { PyScopeManager.Global.Clear(); - + // If the shutdown handlers trigger a domain unload, // don't call shutdown again. AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; ExecuteShutdownHandlers(); + PyObjectConversions.Reset(); + initialized = false; } } @@ -493,7 +494,7 @@ public static PyObject ReloadModule(PyObject module) /// public static PyObject ModuleFromString(string name, string code) { - IntPtr c = Runtime.Py_CompileString(code, "none", (IntPtr)257); + IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); Runtime.CheckExceptionOccurred(); IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); Runtime.CheckExceptionOccurred(); @@ -502,7 +503,7 @@ public static PyObject ModuleFromString(string name, string code) public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) { - var flag = (IntPtr)mode; + var flag = (int)mode; IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); Runtime.CheckExceptionOccurred(); return new PyObject(ptr); @@ -531,12 +532,13 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) { - PyObject result = RunString(code, globals, locals, RunFlagType.File); - if (result.obj != Runtime.PyNone) + using (PyObject result = RunString(code, globals, locals, RunFlagType.File)) { - throw new PythonException(); + if (result.obj != Runtime.PyNone) + { + throw new PythonException(); + } } - result.Dispose(); } @@ -574,7 +576,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, borrowedGlobals = false; } } - + if (locals == null) { locals = globals; @@ -582,13 +584,20 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { - IntPtr result = Runtime.PyRun_String( + NewReference result = Runtime.PyRun_String( code, (IntPtr)flag, globals.Value, locals.Value ); - Runtime.CheckExceptionOccurred(); + try + { + Runtime.CheckExceptionOccurred(); - return new PyObject(result); + return result.MoveToPyObject(); + } + finally + { + result.Dispose(); + } } finally { @@ -600,7 +609,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, } } - public enum RunFlagType + public enum RunFlagType : int { Single = 256, File = 257, /* Py_file_input */ @@ -630,10 +639,11 @@ public static PyScope CreateScope(string name) var scope = PyScopeManager.Global.Create(name); return scope; } - + public class GILState : IDisposable { - private IntPtr state; + private readonly IntPtr state; + private bool isDisposed; internal GILState() { @@ -642,8 +652,11 @@ internal GILState() public void Dispose() { + if (this.isDisposed) return; + PythonEngine.ReleaseLock(state); GC.SuppressFinalize(this); + this.isDisposed = true; } ~GILState() @@ -727,7 +740,7 @@ public static void SetArgv(IEnumerable argv) public static void With(PyObject obj, Action Body) { - // Behavior described here: + // Behavior described here: // https://docs.python.org/2/reference/datamodel.html#with-statement-context-managers IntPtr type = Runtime.PyNone; @@ -744,11 +757,14 @@ public static void With(PyObject obj, Action Body) catch (PythonException e) { ex = e; - type = ex.PyType; - val = ex.PyValue; - traceBack = ex.PyTB; + type = ex.PyType.Coalesce(type); + val = ex.PyValue.Coalesce(val); + traceBack = ex.PyTB.Coalesce(traceBack); } + Runtime.XIncref(type); + Runtime.XIncref(val); + Runtime.XIncref(traceBack); var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); if (ex != null && !exitResult.IsTrue()) throw ex; diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..8efdccc91 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,6 @@ using System; +using System.Runtime.CompilerServices; +using System.Text; namespace Python.Runtime { @@ -6,7 +8,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IPyDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -15,14 +17,12 @@ public class PythonException : System.Exception private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); + Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -45,11 +45,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -60,11 +62,12 @@ public PythonException() ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } /// @@ -132,7 +135,7 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } /// @@ -143,6 +146,47 @@ public string PythonTypeName get { return _pythonTypeName; } } + /// + /// Formats this PythonException object into a message as would be printed + /// out via the Python console. See traceback.format_exception + /// + public string Format() + { + string res; + IntPtr gs = PythonEngine.AcquireLock(); + try + { + if (_pyTB != IntPtr.Zero && _pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + { + Runtime.XIncref(_pyType); + Runtime.XIncref(_pyValue); + Runtime.XIncref(_pyTB); + using (PyObject pyType = new PyObject(_pyType)) + using (PyObject pyValue = new PyObject(_pyValue)) + using (PyObject pyTB = new PyObject(_pyTB)) + using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + { + var buffer = new StringBuilder(); + var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); + foreach (PyObject val in values) + { + buffer.Append(val.ToString()); + } + res = buffer.ToString(); + } + } + else + { + res = StackTrace; + } + } + finally + { + PythonEngine.ReleaseLock(gs); + } + return res; + } + /// /// Dispose Method /// @@ -159,12 +203,23 @@ public void Dispose() if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(_pyType); - Runtime.XDecref(_pyValue); + if (_pyType != IntPtr.Zero) + { + Runtime.XDecref(_pyType); + _pyType= IntPtr.Zero; + } + + if (_pyValue != IntPtr.Zero) + { + Runtime.XDecref(_pyValue); + _pyValue = IntPtr.Zero; + } + // XXX Do we ever get TraceBack? // if (_pyTB != IntPtr.Zero) { Runtime.XDecref(_pyTB); + _pyTB = IntPtr.Zero; } PythonEngine.ReleaseLock(gs); } @@ -173,6 +228,11 @@ public void Dispose() } } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { _pyType, _pyValue, _pyTB }; + } + /// /// Matches Method /// @@ -184,5 +244,21 @@ public static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } + + public static void ThrowIfIsNull(IntPtr ob) + { + if (ob == IntPtr.Zero) + { + throw new PythonException(); + } + } + + public static void ThrowIfIsNotZero(int value) + { + if (value != 0) + { + throw new PythonException(); + } + } } } diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..2254e7430 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "3.0.0dev" class clrproperty(object): diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 863eb4034..e246a19ae 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,102 +1,15 @@ +using System.Reflection.Emit; using System; +using System.Diagnostics.Contracts; using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Threading; using System.Collections.Generic; +using Python.Runtime.Platform; namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } - /// /// Encapsulates the low-level Python C API. Note that it is /// the responsibility of the caller to have acquired the GIL @@ -105,7 +18,7 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public class Runtime { // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static int UCS => _UCS; @@ -130,36 +43,24 @@ public class Runtime #error You must define either UCS2 or UCS4! #endif - // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow - // binary substitution of different Python.Runtime.dll builds in a target application. - - public static string pyversion => _pyversion; - public static string pyver => _pyver; - -#if PYTHON27 - internal const string _pyversion = "2.7"; - internal const string _pyver = "27"; -#elif PYTHON34 - internal const string _pyversion = "3.4"; - internal const string _pyver = "34"; +#if PYTHON34 + const string _minor = "4"; #elif PYTHON35 - internal const string _pyversion = "3.5"; - internal const string _pyver = "35"; + const string _minor = "5"; #elif PYTHON36 - internal const string _pyversion = "3.6"; - internal const string _pyver = "36"; + const string _minor = "6"; #elif PYTHON37 - internal const string _pyversion = "3.7"; - internal const string _pyver = "37"; + const string _minor = "7"; +#elif PYTHON38 + const string _minor = "8"; #else -#error You must define one of PYTHON34 to PYTHON37 or PYTHON27 +#error You must define one of PYTHON34 to PYTHON38 #endif -#if MONO_LINUX || MONO_OSX // Linux/macOS use dotted version string - internal const string dllBase = "python" + _pyversion; -#else // Windows - internal const string dllBase = "python" + _pyver; +#if WINDOWS + internal const string dllBase = "python3" + _minor; +#else + internal const string dllBase = "python3." + _minor; #endif #if PYTHON_WITH_PYDEBUG @@ -174,7 +75,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; @@ -185,28 +86,15 @@ public class Runtime internal const string _PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; #endif - public static readonly int pyversionnumber = Convert.ToInt32(_pyver); - // set to true when python is finalizing internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; - internal static bool Is32Bit = IntPtr.Size == 4; + internal static bool Is32Bit => IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() { { "Windows", OperatingSystemType.Windows }, @@ -214,22 +102,16 @@ public enum OperatingSystemType { "Linux", OperatingSystemType.Linux }, }; - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } + [Obsolete] + public static string OperatingSystemName => OperatingSystem.ToString(); + + [Obsolete] + public static string MachineName => Machine.ToString(); /// /// Gets the operating system as reported by python's platform.system(). /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; + public static OperatingSystemType OperatingSystem { get; private set; } /// /// Map lower-case version of the python machine name to the processor @@ -246,6 +128,9 @@ public enum MachineType ["amd64"] = MachineType.x86_64, ["x64"] = MachineType.x86_64, ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, }; /// @@ -253,19 +138,15 @@ public enum MachineType /// public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - - internal static bool IsPython2 = pyversionnumber < 30; - internal static bool IsPython3 = pyversionnumber >= 30; + public static int MainManagedThreadId { get; private set; } /// /// Encoding to use to convert Unicode to/from Managed to Native /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); + /// /// Initialize the runtime... /// @@ -274,6 +155,7 @@ internal static void Initialize(bool initSigs = false) if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) @@ -283,7 +165,6 @@ internal static void Initialize(bool initSigs = false) IsFinalizing = false; - CLRModule.Reset(); GenericUtil.Reset(); PyScopeManager.Reset(); ClassManager.Reset(); @@ -291,122 +172,126 @@ internal static void Initialize(bool initSigs = false) TypeManager.Reset(); IntPtr op; - IntPtr dict; - if (IsPython3) - { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); - } - else // Python2 { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); + var builtins = GetBuiltins(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented"), + () => PyNotImplemented = IntPtr.Zero); + + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object"), + () => PyBaseObjectType = IntPtr.Zero); + + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None"), + () => PyNone = IntPtr.Zero); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True"), + () => PyTrue = IntPtr.Zero); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), + () => PyFalse = IntPtr.Zero); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + () => PyBoolType = IntPtr.Zero); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + () => PyNoneType = IntPtr.Zero); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + () => PyTypeType = IntPtr.Zero); + + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op), + () => PyMethodType = IntPtr.Zero); + XDecref(op); + + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + () => PyWrapperDescriptorType = IntPtr.Zero); + XDecref(op); + + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super"), + () => PySuper_Type = IntPtr.Zero); + + XDecref(builtins); } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); - - PyModuleType = PyObject_Type(op); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); - - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); - - op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); - XDecref(op); - - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); - XDecref(op); - -#if PYTHON3 - XDecref(dict); -#endif op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op), + () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op), + () => PyUnicodeType = IntPtr.Zero); XDecref(op); -#if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op), + () => PyBytesType = IntPtr.Zero); XDecref(op); -#endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op), + () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op), + () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op), + () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op), + () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op), + () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op), + () => PyFloatType = IntPtr.Zero); + XDecref(op); + + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + SetPyMember(ref PyDecimalType, PyObject_Type(op), + () => PyDecimalType = IntPtr.Zero); XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); -#if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; -#elif PYTHON2 - IntPtr s = PyString_FromString("_temp"); - IntPtr d = PyDict_New(); - - IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); - - IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); - - XDecref(s); - XDecref(i); - XDecref(c); - XDecref(d); -#endif Error = new IntPtr(-1); + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + InitializePlatformData(); + IntPtr dllLocal = IntPtr.Zero; + var loader = LibraryLoader.Get(OperatingSystem); if (_PythonDll != "__Internal") { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); + dllLocal = loader.Load(_PythonDll); } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); + PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); -#if !(MONO_LINUX || MONO_OSX) if (dllLocal != IntPtr.Zero) { - NativeMethods.FreeLibrary(dllLocal); + loader.Free(dllLocal); } -#endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -419,7 +304,7 @@ internal static void Initialize(bool initSigs = false) string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); IntPtr path = PySys_GetObject("path"); IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); + PyList_Append(new BorrowedReference(path), item); XDecref(item); AssemblyManager.UpdatePath(); } @@ -434,6 +319,7 @@ internal static void Initialize(bool initSigs = false) /// private static void InitializePlatformData() { +#if !NETSTANDARD IntPtr op; IntPtr fn; IntPtr platformModule = PyImport_ImportModule("platform"); @@ -441,13 +327,13 @@ private static void InitializePlatformData() fn = PyObject_GetAttrString(platformModule, "system"); op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); + string operatingSystemName = GetManagedString(op); XDecref(op); XDecref(fn); fn = PyObject_GetAttrString(platformModule, "machine"); op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); + string machineName = GetManagedString(op); XDecref(op); XDecref(fn); @@ -457,18 +343,47 @@ private static void InitializePlatformData() // Now convert the strings into enum values so we can do switch // statements rather than constant parsing. OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) + if (!OperatingSystemTypeMapping.TryGetValue(operatingSystemName, out OSType)) { OSType = OperatingSystemType.Other; } OperatingSystem = OSType; MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) + if (!MachineTypeMapping.TryGetValue(machineName.ToLower(), out MType)) { MType = MachineType.Other; } Machine = MType; +#else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + OperatingSystem = OperatingSystemType.Linux; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + OperatingSystem = OperatingSystemType.Darwin; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + OperatingSystem = OperatingSystemType.Windows; + else + OperatingSystem = OperatingSystemType.Other; + + switch (RuntimeInformation.ProcessArchitecture) + { + case Architecture.X86: + Machine = MachineType.i386; + break; + case Architecture.X64: + Machine = MachineType.x86_64; + break; + case Architecture.Arm: + Machine = MachineType.armv7l; + break; + case Architecture.Arm64: + Machine = MachineType.aarch64; + break; + default: + Machine = MachineType.Other; + break; + } +#endif } internal static void Shutdown() @@ -476,6 +391,10 @@ internal static void Shutdown() AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); + Finalizer.Shutdown(); + // TOOD: PyCLRMetaType's release operation still in #958 + PyCLRMetaType = IntPtr.Zero; + ResetPyMembers(); Py_Finalize(); } @@ -489,6 +408,19 @@ internal static int AtExit() return 0; } + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) + { + // XXX: For current usages, value should not be null. + PythonException.ThrowIfIsNull(value); + obj = value; + _pyRefs.Add(value, onRelease); + } + + private static void ResetPyMembers() + { + _pyRefs.Release(); + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; @@ -497,6 +429,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -512,10 +445,11 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; + + internal static IntPtr Py_NoSiteFlag; -#if PYTHON3 internal static IntPtr PyBytesType; -#endif internal static IntPtr _PyObject_NextNotImplemented; internal static IntPtr PyNotImplemented; @@ -531,6 +465,16 @@ internal static int AtExit() internal static IntPtr PyNone; internal static IntPtr Error; + public static PyObject None + { + get + { + var none = Runtime.PyNone; + Runtime.XIncref(none); + return new PyObject(none); + } + } + /// /// Check if any Python Exceptions occurred. /// If any exist throw new PythonException. @@ -661,6 +605,15 @@ internal static unsafe void XIncref(IntPtr op) #endif } + /// + /// Increase Python's ref counter for the given object, and get the object back. + /// + internal static IntPtr SelfIncRef(IntPtr op) + { + XIncref(op); + return op; + } + internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD @@ -698,6 +651,7 @@ internal static unsafe void XDecref(IntPtr op) #endif } + [Pure] internal static unsafe long Refcount(IntPtr op) { var p = (void*)op; @@ -770,16 +724,11 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyGILState_GetThisThreadState(); -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] public static extern int Py_Main( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv ); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - public static extern int Py_Main(int argc, string[] argv); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyEval_InitThreads(); @@ -851,13 +800,36 @@ public static extern int Py_Main( internal static extern int PyRun_SimpleString(string code); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, IntPtr st, IntPtr globals, IntPtr locals); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + /// + internal static IntPtr Py_CompileString(string str, string file, int start) + { + return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + } + + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. + /// + internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + { + return Py_CompileStringExFlags(str, file, start, flags, -1); + } + + /// + /// Return value: New reference. + /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). + /// + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_CompileString(string code, string file, IntPtr tok); + internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); @@ -868,9 +840,6 @@ public static extern int Py_Main( [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyClass_New(IntPtr bases, IntPtr dict, IntPtr name); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyInstance_New(IntPtr cls, IntPtr args, IntPtr kw); @@ -932,11 +901,6 @@ internal static string PyObject_GetTypeName(IntPtr op) internal static bool PyObject_IsIterable(IntPtr pointer) { var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); -#if PYTHON2 - long tp_flags = Util.ReadCLong(ob_type, TypeOffset.tp_flags); - if ((tp_flags & TypeFlags.HaveIter) == 0) - return false; -#endif IntPtr tp_iter = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iter); return tp_iter != IntPtr.Zero; } @@ -977,7 +941,6 @@ internal static bool PyObject_IsIterable(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args); -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid); @@ -1005,10 +968,6 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) Exceptions.SetError(Exceptions.SystemError, "Error comparing objects"); return -1; } -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyObject_Compare(IntPtr value1, IntPtr value2); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_IsInstance(IntPtr ob, IntPtr type); @@ -1027,7 +986,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static long PyObject_Size(IntPtr pointer) { - return (long) _PyObject_Size(pointer); + return (long)_PyObject_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] @@ -1042,14 +1001,9 @@ internal static long PyObject_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Str(IntPtr pointer); -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Str")] internal static extern IntPtr PyObject_Unicode(IntPtr pointer); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyObject_Unicode(IntPtr pointer); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Dir(IntPtr pointer); @@ -1059,14 +1013,9 @@ internal static long PyObject_Size(IntPtr pointer) // Python number API //==================================================================== -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyNumber_Long")] internal static extern IntPtr PyNumber_Int(IntPtr ob); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Int(IntPtr ob); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_Long(IntPtr ob); @@ -1099,7 +1048,6 @@ internal static IntPtr PyInt_FromInt64(long value) return PyInt_FromLong(v); } -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyLong_FromLong")] private static extern IntPtr PyInt_FromLong(IntPtr value); @@ -1112,23 +1060,6 @@ internal static IntPtr PyInt_FromInt64(long value) EntryPoint = "PyLong_FromString")] internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_GetMax")] - internal static extern int PyInt_GetMax(); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyInt_FromLong(IntPtr value); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyInt_AsLong(IntPtr value); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyInt_GetMax(); -#endif - internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyLongType; @@ -1137,8 +1068,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromLong(long value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromUnsignedLong(uint value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong(object value) + { + if (Is32Bit || IsWindows) + return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); + else + return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromDouble(double value); @@ -1155,8 +1099,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern uint PyLong_AsUnsignedLong(IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); + + internal static object PyLong_AsUnsignedLong(IntPtr value) + { + if (Is32Bit || IsWindows) + return PyLong_AsUnsignedLong32(value); + else + return PyLong_AsUnsignedLong64(value); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern long PyLong_AsLongLong(IntPtr value); @@ -1188,7 +1145,7 @@ internal static bool PyFloat_Check(IntPtr ob) internal static extern IntPtr PyNumber_Multiply(IntPtr o1, IntPtr o2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_Divide(IntPtr o1, IntPtr o2); + internal static extern IntPtr PyNumber_TrueDivide(IntPtr o1, IntPtr o2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_And(IntPtr o1, IntPtr o2); @@ -1221,7 +1178,7 @@ internal static bool PyFloat_Check(IntPtr ob) internal static extern IntPtr PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyNumber_InPlaceDivide(IntPtr o1, IntPtr o2); + internal static extern IntPtr PyNumber_InPlaceTrueDivide(IntPtr o1, IntPtr o2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2); @@ -1311,7 +1268,7 @@ internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) internal static long PySequence_Size(IntPtr pointer) { - return (long) _PySequence_Size(pointer); + return (long)_PySequence_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] @@ -1336,7 +1293,7 @@ internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) internal static long PySequence_Count(IntPtr pointer, IntPtr value) { - return (long) _PySequence_Count(pointer, value); + return (long)_PySequence_Count(pointer, value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] @@ -1366,20 +1323,15 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { -#if PYTHON3 return PyUnicode_FromKindAndData(_UCS, value, value.Length); -#elif PYTHON2 - return PyString_FromStringAndSize(value, value.Length); -#endif } -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); internal static long PyBytes_Size(IntPtr op) { - return (long) _PyBytes_Size(op); + return (long)_PyBytes_Size(op); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] @@ -1409,28 +1361,15 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); -#elif PYTHON2 - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return PyString_FromStringAndSize(value, new IntPtr(size)); - } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyString_FromStringAndSize(string value, IntPtr size); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyString_AsString(IntPtr op); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyString_Size(IntPtr pointer); -#endif + internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); internal static bool PyUnicode_Check(IntPtr ob) { return PyObject_TYPE(ob) == PyUnicodeType; } -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); @@ -1467,50 +1406,14 @@ internal static long PyUnicode_GetSize(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromOrdinal(int c); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "FromObject")] - internal static extern IntPtr PyUnicode_FromObject(IntPtr ob); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] - internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromUnicode(s, new IntPtr(size)); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - private static extern IntPtr PyUnicode_FromUnicode( - [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size - ); - - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long) _PyUnicode_GetSize(ob); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern IntPtr _PyUnicode_GetSize(IntPtr ob); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] - internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = PyUnicodeEntryPoint + "FromOrdinal")] - internal static extern IntPtr PyUnicode_FromOrdinal(int c); -#endif internal static IntPtr PyUnicode_FromString(string s) { return PyUnicode_FromUnicode(s, s.Length); } + internal static string GetManagedString(in BorrowedReference borrowedReference) + => GetManagedString(borrowedReference.DangerousGetAddress()); /// /// Function to access the internal PyUnicode/PyString object and /// convert it to a managed string with the correct encoding. @@ -1528,13 +1431,6 @@ internal static string GetManagedString(IntPtr op) { IntPtr type = PyObject_TYPE(op); -#if PYTHON2 // Python 3 strings are all Unicode - if (type == PyStringType) - { - return Marshal.PtrToStringAnsi(PyString_AsString(op), PyString_Size(op)); - } -#endif - if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); @@ -1593,7 +1489,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static extern IntPtr PyDict_Values(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Items(IntPtr pointer); + internal static extern NewReference PyDict_Items(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Copy(IntPtr pointer); @@ -1606,7 +1502,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static long PyDict_Size(IntPtr pointer) { - return (long) _PyDict_Size(pointer); + return (long)_PyDict_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] @@ -1633,13 +1529,13 @@ internal static IntPtr PyList_New(long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) + internal static BorrowedReference PyList_GetItem(IntPtr pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); + private static extern BorrowedReference PyList_GetItem(IntPtr pointer, IntPtr index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) { @@ -1649,22 +1545,22 @@ internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) + internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) { return PyList_Insert(pointer, new IntPtr(index), value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + private static extern int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Append(IntPtr pointer, IntPtr value); + internal static extern int PyList_Append(BorrowedReference pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Reverse(IntPtr pointer); + internal static extern int PyList_Reverse(BorrowedReference pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Sort(IntPtr pointer); + internal static extern int PyList_Sort(BorrowedReference pointer); internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) { @@ -1684,7 +1580,7 @@ internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr internal static long PyList_Size(IntPtr pointer) { - return (long) _PyList_Size(pointer); + return (long)_PyList_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] @@ -1733,7 +1629,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static long PyTuple_Size(IntPtr pointer) { - return (long) _PyTuple_Size(pointer); + return (long)_PyTuple_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] @@ -1747,11 +1643,6 @@ internal static long PyTuple_Size(IntPtr pointer) internal static bool PyIter_Check(IntPtr pointer) { var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); -#if PYTHON2 - long tp_flags = Util.ReadCLong(ob_type, TypeOffset.tp_flags); - if ((tp_flags & TypeFlags.HaveIter) == 0) - return false; -#endif IntPtr tp_iternext = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iternext); return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } @@ -1776,14 +1667,15 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern string PyModule_GetFilename(IntPtr module); -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_Import(IntPtr name); + /// + /// Return value: New reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ImportModule(string name); @@ -1796,21 +1688,12 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_GetModuleDict(); -#if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PySys_SetArgvEx( int argc, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, int updatepath ); -#elif PYTHON2 - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PySys_SetArgvEx( - int argc, - string[] argv, - int updatepath - ); -#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySys_GetObject(string name); @@ -1839,6 +1722,10 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) IntPtr t = PyObject_TYPE(ob); return (t == tp) || PyType_IsSubtype(t, tp); } + internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) + { + return (type == ofType) || PyType_IsSubtype(type, ofType); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); @@ -1932,7 +1819,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern IntPtr PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); + internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); @@ -1943,6 +1830,15 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + //==================================================================== + // Cell API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCell_Get(BorrowedReference cell); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); //==================================================================== // Miscellaneous @@ -1953,5 +1849,66 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); + + internal static void SetNoSiteFlag() + { + var loader = LibraryLoader.Get(OperatingSystem); + + IntPtr dllLocal = _PythonDll != "__Internal" + ? loader.Load(_PythonDll) + : IntPtr.Zero; + + try + { + Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); + Marshal.WriteInt32(Py_NoSiteFlag, 1); + } + finally + { + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } + } + } + + /// + /// Return value: New reference. + /// + internal static IntPtr GetBuiltins() + { + return PyImport_ImportModule("builtins"); + } + } + + + class PyReferenceCollection + { + private List> _actions = new List>(); + + /// + /// Record obj's address to release the obj in the future, + /// obj must alive before calling Release. + /// + public void Add(IntPtr ob, Action onRelease) + { + _actions.Add(new KeyValuePair(ob, onRelease)); + } + + public void Release() + { + foreach (var item in _actions) + { + Runtime.XDecref(item.Key); + item.Value?.Invoke(); + } + _actions.Clear(); + } } } diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs new file mode 100644 index 000000000..b0a2e8d79 --- /dev/null +++ b/src/runtime/slots/mp_length.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Python.Runtime.Slots +{ + internal static class mp_length_slot + { + /// + /// Implements __len__ for classes that implement ICollection + /// (this includes any IList implementer or Array subclass) + /// + public static int mp_length(IntPtr ob) + { + var co = ManagedType.GetManagedObject(ob) as CLRObject; + if (co == null) + { + Exceptions.RaiseTypeError("invalid object"); + } + + // first look for ICollection implementation directly + if (co.inst is ICollection c) + { + return c.Count; + } + + Type clrType = co.inst.GetType(); + + // now look for things that implement ICollection directly (non-explicitly) + PropertyInfo p = clrType.GetProperty("Count"); + if (p != null && clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return (int)p.GetValue(co.inst, null); + } + + // finally look for things that implement the interface explicitly + var iface = clrType.GetInterfaces().FirstOrDefault(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>)); + if (iface != null) + { + p = iface.GetProperty(nameof(ICollection.Count)); + return (int)p.GetValue(co.inst, null); + } + + Exceptions.SetError(Exceptions.TypeError, $"object of type '{clrType.Name}' has no len()"); + return -1; + } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..04d40a2ba 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,11 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using Python.Runtime.Platform; +using Python.Runtime.Slots; namespace Python.Runtime { + /// /// The TypeManager class is responsible for building binary-compatible /// Python type objects that are implemented in managed code. @@ -83,7 +87,7 @@ internal static IntPtr CreateType(Type impl) // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.DictOffset(type); + var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); InitializeSlots(type, impl); @@ -121,7 +125,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) IntPtr base_ = IntPtr.Zero; int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* @@ -129,9 +132,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (typeof(Exception).IsAssignableFrom(clrType)) { ob_size = ObjectOffset.Size(Exceptions.Exception); - tp_dictoffset = ObjectOffset.DictOffset(Exceptions.Exception); } + int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; + if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -151,6 +155,13 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // add a __len__ slot for inheritors of ICollection and ICollection<> + if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + } + + // we want to do this after the slot stuff above in case the class itself implements a slot method InitializeSlots(type, impl.GetType()); if (base_ != IntPtr.Zero) @@ -191,6 +202,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + } + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -263,6 +280,14 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); Runtime.PyDict_Update(cls_dict, py_dict); + // Update the __classcell__ if it exists + var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); + if (!cell.IsNull) + { + Runtime.PyCell_Set(cell, py_type); + Runtime.PyDict_DelItemString(cls_dict, "__classcell__"); + } + return py_type; } catch (Exception e) @@ -307,17 +332,11 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - // Copy gc and other type slots from the base Python metatype. - - CopySlot(py_type, type, TypeOffset.tp_basicsize); - CopySlot(py_type, type, TypeOffset.tp_itemsize); - - CopySlot(py_type, type, TypeOffset.tp_dictoffset); - CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); - - CopySlot(py_type, type, TypeOffset.tp_traverse); - CopySlot(py_type, type, TypeOffset.tp_clear); - CopySlot(py_type, type, TypeOffset.tp_is_gc); + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. // Override type slots with those of the managed implementation. @@ -333,16 +352,18 @@ internal static IntPtr CreateMetaType(Type impl) // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); IntPtr mdefStart = mdef; + ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__instancecheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc") + thunkInfo.Address ); + thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__subclasscheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc") + thunkInfo.Address ); // FIXME: mdef is not used @@ -412,23 +433,12 @@ internal static IntPtr AllocateTypeObject(string name) // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. -#if PYTHON3 - // For python3 we leak two objects. One for the ASCII representation - // required for tp_name, and another for the Unicode representation - // for ht_name. - IntPtr temp = Runtime.PyBytes_FromString(name); - IntPtr raw = Runtime.PyBytes_AS_STRING(temp); - temp = Runtime.PyUnicode_FromString(name); -#elif PYTHON2 - IntPtr temp = Runtime.PyString_FromString(name); - IntPtr raw = Runtime.PyString_AsString(temp); -#endif + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); -#if PYTHON3 Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); -#endif long ptr = type.ToInt64(); // 64-bit safe @@ -441,11 +451,7 @@ internal static IntPtr AllocateTypeObject(string name) temp = new IntPtr(ptr + TypeOffset.mp_length); Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); -#if PYTHON3 temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); -#elif PYTHON2 - temp = new IntPtr(ptr + TypeOffset.bf_getreadbuffer); -#endif Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); return type; } @@ -504,14 +510,14 @@ public static NativeCode Active { get { - switch(Runtime.Machine) + switch (Runtime.Machine) { - case Runtime.MachineType.i386: + case MachineType.i386: return I386; - case Runtime.MachineType.x86_64: + case MachineType.x86_64: return X86_64; default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); + return null; } } } @@ -603,12 +609,14 @@ int MAP_ANONYMOUS { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: + case OperatingSystemType.Darwin: return 0x1000; - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Linux: return 0x20; default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); + throw new NotImplementedException( + $"mmap is not supported on {Runtime.OperatingSystem}" + ); } } } @@ -636,13 +644,15 @@ internal static IMemoryMapper CreateMemoryMapper() { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: + case OperatingSystemType.Windows: return new WindowsMemoryMapper(); default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); + throw new NotImplementedException( + $"No support for {Runtime.OperatingSystem}" + ); } } @@ -668,7 +678,7 @@ internal static void InitializeNativeCodePage() Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); mapper.SetReadExec(NativeCodePage, codeLength); } -#endregion + #endregion /// /// Given a newly allocated Python type object and a managed Type that @@ -703,7 +713,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + var thunkInfo = Interop.GetThunk(method); + InitializeSlot(type, thunkInfo.Address, name); seen.Add(name); } @@ -711,21 +722,40 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: + var native = NativeCode.Active; + + // The garbage collection related slots always have to return 1 or 0 + // since .NET objects don't take part in Python's gc: // tp_traverse (returns 0) // tp_clear (returns 0) // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); + // These have to be defined, though, so by default we fill these with + // static C# functions from this class. + + var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; + var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; + + if (native != null) + { + // If we want to support domain reload, the C# implementation + // cannot be used as the assembly may get released before + // CPython calls these functions. Instead, for amd64 and x86 we + // load them into a separate code page that is leaked + // intentionally. + InitializeNativeCodePage(); + ret1 = NativeCodePage + native.Return1; + ret0 = NativeCodePage + native.Return0; + } + + InitializeSlot(type, ret0, "tp_traverse"); + InitializeSlot(type, ret0, "tp_clear"); + InitializeSlot(type, ret1, "tp_is_gc"); } + static int Return1(IntPtr _) => 1; + + static int Return0(IntPtr _) => 0; + /// /// Helper for InitializeSlots. /// diff --git a/src/testing/Python.Test.15.csproj b/src/testing/Python.Test.15.csproj index da20ed2ef..0e19adf91 100644 --- a/src/testing/Python.Test.15.csproj +++ b/src/testing/Python.Test.15.csproj @@ -7,7 +7,7 @@ Python.Test Python.Test Python.Test - 2.4.0 + 2.5.0 bin\ false $(OutputPath)\$(AssemblyName).xml diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..98df48e6b 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Test bin\Python.Test.xml bin\ - v4.0 + v4.5.2 1591,0067 ..\..\ @@ -70,6 +70,7 @@ pdbonly + @@ -91,6 +92,9 @@ + + + diff --git a/src/testing/ReprTest.cs b/src/testing/ReprTest.cs new file mode 100644 index 000000000..48e93683a --- /dev/null +++ b/src/testing/ReprTest.cs @@ -0,0 +1,108 @@ +using System; +using System.Text; + +namespace Python.Test +{ + /// + /// Supports repr unit tests. + /// + public class ReprTest + { + public class Point + { + public Point(double x, double y) + { + X = x; + Y = y; + } + + public double X { get; set; } + public double Y { get; set; } + + public override string ToString() + { + return base.ToString() + ": X=" + X.ToString() + ", Y=" + Y.ToString(); + } + + public string __repr__() + { + return "Point(" + X.ToString() + "," + Y.ToString() + ")"; + } + } + + public class Foo + { + public string __repr__() + { + return "I implement __repr__() but not ToString()!"; + } + } + + public class Bar + { + public override string ToString() + { + return "I implement ToString() but not __repr__()!"; + } + } + + public class BazBase + { + public override string ToString() + { + return "Base class implementing ToString()!"; + } + } + + public class BazMiddle : BazBase + { + public override string ToString() + { + return "Middle class implementing ToString()!"; + } + } + + //implements ToString via BazMiddle + public class Baz : BazMiddle + { + + } + + public class Quux + { + public string ToString(string format) + { + return "I implement ToString() with an argument!"; + } + } + + public class QuuzBase + { + protected string __repr__() + { + return "I implement __repr__ but it isn't public!"; + } + } + + public class Quuz : QuuzBase + { + + } + + public class Corge + { + public string __repr__(int i) + { + return "__repr__ implemention with input parameter!"; + } + } + + public class Grault + { + public int __repr__() + { + return "__repr__ implemention with wrong return type!".Length; + } + } + } +} diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..1f9d64e1b 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -1,7 +1,9 @@ namespace Python.Test { + using System.Collections.Generic; + /// - /// Supports units tests for field access. + /// Supports unit tests for field access. /// public class ConversionTest { @@ -32,6 +34,7 @@ public ConversionTest() public byte[] ByteArrayField; public sbyte[] SByteArrayField; + public readonly List ListField = new List(); public T? Echo(T? arg) where T: struct { return arg; diff --git a/src/testing/dictionarytest.cs b/src/testing/dictionarytest.cs new file mode 100644 index 000000000..a7fa3497d --- /dev/null +++ b/src/testing/dictionarytest.cs @@ -0,0 +1,106 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace Python.Test +{ + /// + /// Supports units tests for dictionary __contains__ and __len__ + /// + public class PublicDictionaryTest + { + public IDictionary items; + + public PublicDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class ProtectedDictionaryTest + { + protected IDictionary items; + + public ProtectedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class InternalDictionaryTest + { + internal IDictionary items; + + public InternalDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class PrivateDictionaryTest + { + private IDictionary items; + + public PrivateDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + public class InheritedDictionaryTest : IDictionary + { + private readonly IDictionary items; + + public InheritedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + + public int this[string key] + { + get { return items[key]; } + set { items[key] = value; } + } + + public ICollection Keys => items.Keys; + + public ICollection Values => items.Values; + + public int Count => items.Count; + + public bool IsReadOnly => false; + + public void Add(string key, int value) => items.Add(key, value); + + public void Add(KeyValuePair item) => items.Add(item); + + public void Clear() => items.Clear(); + + public bool Contains(KeyValuePair item) => items.Contains(item); + + public bool ContainsKey(string key) => items.ContainsKey(key); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator> GetEnumerator() => items.GetEnumerator(); + + public bool Remove(string key) => items.Remove(key); + + public bool Remove(KeyValuePair item) => items.Remove(item); + + public bool TryGetValue(string key, out int value) => items.TryGetValue(key, out value); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index cf653f9f9..91836b727 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Runtime.InteropServices; namespace Python.Test { @@ -651,6 +652,38 @@ public static string Casesensitive() { return "Casesensitive"; } + + public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static bool OptionalParams_TestMissing([Optional]object a) + { + return a == Type.Missing; + } + + public static bool OptionalParams_TestReferenceType([Optional]string a) + { + return a == null; + } + + public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + } diff --git a/src/testing/mp_lengthtest.cs b/src/testing/mp_lengthtest.cs new file mode 100644 index 000000000..a4f3e8c25 --- /dev/null +++ b/src/testing/mp_lengthtest.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Python.Test +{ + public class MpLengthCollectionTest : ICollection + { + private readonly List items; + + public MpLengthCollectionTest() + { + SyncRoot = new object(); + items = new List + { + 1, + 2, + 3 + }; + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public void CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthExplicitCollectionTest : ICollection + { + private readonly List items; + private readonly object syncRoot; + + public MpLengthExplicitCollectionTest() + { + syncRoot = new object(); + items = new List + { + 9, + 10 + }; + } + int ICollection.Count => items.Count; + + object ICollection.SyncRoot => syncRoot; + + bool ICollection.IsSynchronized => false; + + void ICollection.CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthGenericCollectionTest() { + SyncRoot = new object(); + items = new List(); + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public bool IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + public void Clear() + { + items.Clear(); + } + + public bool Contains(T item) + { + return items.Contains(item); + } + + public void CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + public bool Remove(T item) + { + return items.Remove(item); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + } + + public class MpLengthExplicitGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthExplicitGenericCollectionTest() + { + items = new List(); + } + + int ICollection.Count => items.Count; + + bool ICollection.IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + void ICollection.Clear() + { + items.Clear(); + } + + bool ICollection.Contains(T item) + { + return items.Contains(item); + } + + void ICollection.CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + bool ICollection.Remove(T item) + { + return items.Remove(item); + } + } +} diff --git a/src/testing/nonexportable.cs b/src/testing/nonexportable.cs new file mode 100644 index 000000000..a29c78589 --- /dev/null +++ b/src/testing/nonexportable.cs @@ -0,0 +1,8 @@ +namespace Python.Test +{ + using Python.Runtime; + + // this class should not be visible to Python + [PyExport(false)] + public class NonExportable { } +} diff --git a/src/tests/_compat.py b/src/tests/_compat.py deleted file mode 100644 index 3751ca013..000000000 --- a/src/tests/_compat.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Python 2.7, 3.3+ compatibility module. - -Using Python 3 syntax to encourage upgrade unless otherwise noted. -""" - -import operator -import subprocess -import sys -import types - -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY3: - import _thread as thread # Using PY2 name - import pickle - from collections import UserList - - indexbytes = operator.getitem - input = input - - string_types = str, - binary_type = bytes - text_type = str - - DictProxyType = type(object.__dict__) - ClassType = type - - # No PY3 equivalents, use PY2 name - long = int - unichr = chr - unicode = str - - # from nowhere import Nothing - cmp = lambda a, b: (a > b) - (a < b) # No PY3 equivalent - map = map - range = range - zip = zip - -elif PY2: - import thread # Using PY2 name - import cPickle as pickle - from UserList import UserList - - indexbytes = lambda buf, i: ord(buf[i]) - input = raw_input - - string_types = str, unicode - bytes_type = str - text_type = unicode - - DictProxyType = types.DictProxyType - ClassType = types.ClassType - - # No PY3 equivalents, use PY2 name - long = long - unichr = unichr - unicode = unicode - - from itertools import izip, imap - cmp = cmp - map = imap - range = xrange - zip = izip - - -def check_output(*args, **kwargs): - """Check output wrapper for PY2/PY3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY2: - return output - return output.decode("ascii") diff --git a/src/tests/leaktest.py b/src/tests/leaktest.py index 05b76e867..02133fece 100644 --- a/src/tests/leaktest.py +++ b/src/tests/leaktest.py @@ -10,7 +10,6 @@ import System -from ._compat import range from .utils import (CallableHandler, ClassMethodHandler, GenericHandler, HelloClass, StaticMethodHandler, VarCallableHandler, VariableArgsHandler, hello_func) diff --git a/src/tests/profile.py b/src/tests/profile.py index 4af3589e8..2113b1727 100644 --- a/src/tests/profile.py +++ b/src/tests/profile.py @@ -14,7 +14,6 @@ import time import runtests -from ._compat import range def main(): diff --git a/src/tests/runtests.py b/src/tests/runtests.py index 8011d05e6..9b90bcf6a 100644 --- a/src/tests/runtests.py +++ b/src/tests/runtests.py @@ -8,8 +8,6 @@ import sys import pytest -from ._compat import input - try: import System except ImportError: diff --git a/src/tests/stress.py b/src/tests/stress.py index c6fa8b7e3..f1f0fac6b 100644 --- a/src/tests/stress.py +++ b/src/tests/stress.py @@ -17,7 +17,7 @@ import threading import time -from ._compat import range, thread +import _thread as thread from .utils import dprint diff --git a/src/tests/stresstest.py b/src/tests/stresstest.py index 74b863bdc..b0dca9461 100644 --- a/src/tests/stresstest.py +++ b/src/tests/stresstest.py @@ -11,8 +11,6 @@ import unittest # import pdb -from ._compat import range - try: import System except ImportError: diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 7ccadddff..427958ec7 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -6,7 +6,7 @@ import System import pytest -from ._compat import PY2, UserList, long, range, unichr +from collections import UserList def test_public_array(): @@ -246,8 +246,8 @@ def test_char_array(): assert items[0] == 'a' assert items[4] == 'e' - max_ = unichr(65535) - min_ = unichr(0) + max_ = chr(65535) + min_ = chr(0) items[0] = max_ assert items[0] == max_ @@ -364,8 +364,8 @@ def test_int64_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(9223372036854775807) - min_ = long(-9223372036854775808) + max_ = 9223372036854775807 + min_ = -9223372036854775808 items[0] = max_ assert items[0] == max_ @@ -448,7 +448,7 @@ def test_uint32_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(4294967295) + max_ = 4294967295 min_ = 0 items[0] = max_ @@ -490,7 +490,7 @@ def test_uint64_array(): assert items[0] == 0 assert items[4] == 4 - max_ = long(18446744073709551615) + max_ = 18446744073709551615 min_ = 0 items[0] = max_ @@ -1204,8 +1204,8 @@ def test_special_array_creation(): assert value.Length == 2 value = Array[System.Char]([0, 65535]) - assert value[0] == unichr(0) - assert value[1] == unichr(65535) + assert value[0] == chr(0) + assert value[1] == chr(65535) assert value.Length == 2 value = Array[System.Int16]([0, 32767]) @@ -1223,31 +1223,24 @@ def test_special_array_creation(): assert value[1] == 2147483647 assert value.Length == 2 - value = Array[System.Int64]([0, long(9223372036854775807)]) + value = Array[System.Int64]([0, 9223372036854775807]) assert value[0] == 0 - assert value[1] == long(9223372036854775807) + assert value[1] == 9223372036854775807 assert value.Length == 2 - # there's no explicit long type in python3, use System.Int64 instead - if PY2: - value = Array[long]([0, long(9223372036854775807)]) - assert value[0] == 0 - assert value[1] == long(9223372036854775807) - assert value.Length == 2 - value = Array[System.UInt16]([0, 65000]) assert value[0] == 0 assert value[1] == 65000 assert value.Length == 2 - value = Array[System.UInt32]([0, long(4294967295)]) + value = Array[System.UInt32]([0, 4294967295]) assert value[0] == 0 - assert value[1] == long(4294967295) + assert value[1] == 4294967295 assert value.Length == 2 - value = Array[System.UInt64]([0, long(18446744073709551615)]) + value = Array[System.UInt64]([0, 18446744073709551615]) assert value[0] == 0 - assert value[1] == long(18446744073709551615) + assert value[1] == 18446744073709551615 assert value.Length == 2 value = Array[System.Single]([0.0, 3.402823e38]) @@ -1337,3 +1330,32 @@ def test_array_abuse(): with pytest.raises(TypeError): desc = Test.PublicArrayTest.__dict__['__setitem__'] desc(0, 0, 0) + + +def test_iterator_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + keys_iterator = iter(d.keys()) + arr = Array[String](keys_iterator) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + + +def test_dict_keys_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + d_keys = d.keys() + arr = Array[String](d_keys) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..2f15f35b1 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -7,14 +7,13 @@ import System import pytest -from ._compat import DictProxyType, range +from .utils import DictProxyType def test_basic_reference_type(): """Test usage of CLR defined reference types.""" assert System.String.Empty == "" - def test_basic_value_type(): """Test usage of CLR defined value types.""" assert System.Int32.MaxValue == 2147483647 @@ -29,7 +28,6 @@ def test_class_standard_attrs(): assert isinstance(ClassTest.__dict__, DictProxyType) assert len(ClassTest.__doc__) > 0 - def test_class_docstrings(): """Test standard class docstring generation""" from Python.Test import ClassTest @@ -58,6 +56,14 @@ def test_non_public_class(): with pytest.raises(AttributeError): _ = Test.InternalClass +def test_non_exported(): + """Test [PyExport(false)]""" + with pytest.raises(ImportError): + from Python.Test import NonExportable + + with pytest.raises(AttributeError): + _ = Test.NonExportable + def test_basic_subclass(): """Test basic subclass of a managed class.""" diff --git a/src/tests/test_compat.py b/src/tests/test_compat.py index 81e7f8143..1c9f80e65 100644 --- a/src/tests/test_compat.py +++ b/src/tests/test_compat.py @@ -7,7 +7,6 @@ import pytest -from ._compat import ClassType, PY2, PY3, range from .utils import is_clr_class, is_clr_module, is_clr_root_module @@ -22,15 +21,9 @@ def test_simple_import(): assert isinstance(sys, types.ModuleType) assert sys.__name__ == 'sys' - if PY3: - import http.client - assert isinstance(http.client, types.ModuleType) - assert http.client.__name__ == 'http.client' - - elif PY2: - import httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'httplib' + import http.client + assert isinstance(http.client, types.ModuleType) + assert http.client.__name__ == 'http.client' def test_simple_import_with_alias(): @@ -43,15 +36,9 @@ def test_simple_import_with_alias(): assert isinstance(mySys, types.ModuleType) assert mySys.__name__ == 'sys' - if PY3: - import http.client as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'http.client' - - elif PY2: - import httplib as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'httplib' + import http.client as myHttplib + assert isinstance(myHttplib, types.ModuleType) + assert myHttplib.__name__ == 'http.client' def test_dotted_name_import(): @@ -125,7 +112,7 @@ def test_dotted_name_import_from(): assert pulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM - assert isinstance(PullDOM, ClassType) + assert isinstance(PullDOM, type) assert PullDOM.__name__ == 'PullDOM' @@ -144,7 +131,7 @@ def test_dotted_name_import_from_with_alias(): assert myPulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM as myPullDOM - assert isinstance(myPullDOM, ClassType) + assert isinstance(myPullDOM, type) assert myPullDOM.__name__ == 'PullDOM' diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..74613abd1 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - """Test CLR <-> Python type conversions.""" -from __future__ import unicode_literals -import System +import operator import pytest -from Python.Test import ConversionTest, UnicodeString -from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 +import System +from Python.Test import ConversionTest, UnicodeString +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder def test_bool_conversion(): @@ -143,8 +142,8 @@ def test_byte_conversion(): def test_char_conversion(): """Test char conversion.""" - assert System.Char.MaxValue == unichr(65535) - assert System.Char.MinValue == unichr(0) + assert System.Char.MaxValue == chr(65535) + assert System.Char.MinValue == chr(0) ob = ConversionTest() assert ob.CharField == u'A' @@ -248,23 +247,23 @@ def test_int32_conversion(): def test_int64_conversion(): """Test int64 conversion.""" - assert System.Int64.MaxValue == long(9223372036854775807) - assert System.Int64.MinValue == long(-9223372036854775808) + assert System.Int64.MaxValue == 9223372036854775807 + assert System.Int64.MinValue == -9223372036854775808 ob = ConversionTest() assert ob.Int64Field == 0 - ob.Int64Field = long(9223372036854775807) - assert ob.Int64Field == long(9223372036854775807) + ob.Int64Field = 9223372036854775807 + assert ob.Int64Field == 9223372036854775807 - ob.Int64Field = long(-9223372036854775808) - assert ob.Int64Field == long(-9223372036854775808) + ob.Int64Field = -9223372036854775808 + assert ob.Int64Field == -9223372036854775808 - ob.Int64Field = System.Int64(long(9223372036854775807)) - assert ob.Int64Field == long(9223372036854775807) + ob.Int64Field = System.Int64(9223372036854775807) + assert ob.Int64Field == 9223372036854775807 - ob.Int64Field = System.Int64(long(-9223372036854775808)) - assert ob.Int64Field == long(-9223372036854775808) + ob.Int64Field = System.Int64(-9223372036854775808) + assert ob.Int64Field == -9223372036854775808 with pytest.raises(TypeError): ConversionTest().Int64Field = "spam" @@ -273,16 +272,16 @@ def test_int64_conversion(): ConversionTest().Int64Field = None with pytest.raises(OverflowError): - ConversionTest().Int64Field = long(9223372036854775808) + ConversionTest().Int64Field = 9223372036854775808 with pytest.raises(OverflowError): - ConversionTest().Int64Field = long(-9223372036854775809) + ConversionTest().Int64Field = -9223372036854775809 with pytest.raises(OverflowError): - _ = System.Int64(long(9223372036854775808)) + _ = System.Int64(9223372036854775808) with pytest.raises(OverflowError): - _ = System.Int64(long(-9223372036854775809)) + _ = System.Int64(-9223372036854775809) def test_uint16_conversion(): @@ -326,20 +325,20 @@ def test_uint16_conversion(): def test_uint32_conversion(): """Test uint32 conversion.""" - assert System.UInt32.MaxValue == long(4294967295) + assert System.UInt32.MaxValue == 4294967295 assert System.UInt32.MinValue == 0 ob = ConversionTest() assert ob.UInt32Field == 0 - ob.UInt32Field = long(4294967295) - assert ob.UInt32Field == long(4294967295) + ob.UInt32Field = 4294967295 + assert ob.UInt32Field == 4294967295 ob.UInt32Field = -0 assert ob.UInt32Field == 0 - ob.UInt32Field = System.UInt32(long(4294967295)) - assert ob.UInt32Field == long(4294967295) + ob.UInt32Field = System.UInt32(4294967295) + assert ob.UInt32Field == 4294967295 ob.UInt32Field = System.UInt32(0) assert ob.UInt32Field == 0 @@ -351,13 +350,13 @@ def test_uint32_conversion(): ConversionTest().UInt32Field = None with pytest.raises(OverflowError): - ConversionTest().UInt32Field = long(4294967296) + ConversionTest().UInt32Field = 4294967296 with pytest.raises(OverflowError): ConversionTest().UInt32Field = -1 with pytest.raises(OverflowError): - _ = System.UInt32(long(4294967296)) + _ = System.UInt32(4294967296) with pytest.raises(OverflowError): _ = System.UInt32(-1) @@ -365,20 +364,20 @@ def test_uint32_conversion(): def test_uint64_conversion(): """Test uint64 conversion.""" - assert System.UInt64.MaxValue == long(18446744073709551615) + assert System.UInt64.MaxValue == 18446744073709551615 assert System.UInt64.MinValue == 0 ob = ConversionTest() assert ob.UInt64Field == 0 - ob.UInt64Field = long(18446744073709551615) - assert ob.UInt64Field == long(18446744073709551615) + ob.UInt64Field = 18446744073709551615 + assert ob.UInt64Field == 18446744073709551615 ob.UInt64Field = -0 assert ob.UInt64Field == 0 - ob.UInt64Field = System.UInt64(long(18446744073709551615)) - assert ob.UInt64Field == long(18446744073709551615) + ob.UInt64Field = System.UInt64(18446744073709551615) + assert ob.UInt64Field == 18446744073709551615 ob.UInt64Field = System.UInt64(0) assert ob.UInt64Field == 0 @@ -390,13 +389,13 @@ def test_uint64_conversion(): ConversionTest().UInt64Field = None with pytest.raises(OverflowError): - ConversionTest().UInt64Field = long(18446744073709551616) + ConversionTest().UInt64Field = 18446744073709551616 with pytest.raises(OverflowError): ConversionTest().UInt64Field = -1 with pytest.raises(OverflowError): - _ = System.UInt64(long(18446744073709551616)) + _ = System.UInt64((18446744073709551616)) with pytest.raises(OverflowError): _ = System.UInt64(-1) @@ -476,7 +475,7 @@ def test_decimal_conversion(): max_d = Decimal.Parse("79228162514264337593543950335") min_d = Decimal.Parse("-79228162514264337593543950335") - assert Decimal.ToInt64(Decimal(10)) == long(10) + assert Decimal.ToInt64(Decimal(10)) == 10 ob = ConversionTest() assert ob.DecimalField == Decimal(0) @@ -536,14 +535,12 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 - + world = UnicodeString() test_unicode_str = u"안녕" - assert test_unicode_str == text_type(world.value) - assert test_unicode_str == text_type(world.GetString()) - # TODO: not sure what to do for Python 2 here (GH PR #670) - if PY3: - assert test_unicode_str == text_type(world) + assert test_unicode_str == str(world.value) + assert test_unicode_str == str(world.GetString()) + assert test_unicode_str == str(world) def test_interface_conversion(): @@ -595,11 +592,10 @@ def test_object_conversion(): # need to test subclass here - with pytest.raises(TypeError): - class Foo(object): - pass - ob = ConversionTest() - ob.ObjectField = Foo + class Foo(object): + pass + ob.ObjectField = Foo + assert ob.ObjectField == Foo def test_enum_conversion(): @@ -640,7 +636,7 @@ def test_enum_conversion(): def test_null_conversion(): """Test null conversion.""" import System - + ob = ConversionTest() ob.StringField = None @@ -681,7 +677,7 @@ def test_byte_array_conversion(): ob.ByteArrayField = value array = ob.ByteArrayField for i, _ in enumerate(value): - assert array[i] == indexbytes(value, i) + assert array[i] == operator.getitem(value, i) def test_sbyte_array_conversion(): @@ -700,4 +696,20 @@ def test_sbyte_array_conversion(): ob.SByteArrayField = value array = ob.SByteArrayField for i, _ in enumerate(value): - assert array[i] == indexbytes(value, i) + assert array[i] == operator.getitem(value, i) + +def test_codecs(): + """Test codec registration from Python""" + class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = "Python.Test" + def CanEncode(self, clr_type): + return clr_type.Name == "List`1" and clr_type.Namespace == "System.Collections.Generic" + + list_raw_encoder = ListAsRawEncoder() + PyObjectConversions.RegisterEncoder(list_raw_encoder) + + ob = ConversionTest() + + l = ob.ListField + l.Add(42) + assert ob.ListField.Count == 1 diff --git a/src/tests/test_delegate.py b/src/tests/test_delegate.py index 33aca43b3..1bfc4e903 100644 --- a/src/tests/test_delegate.py +++ b/src/tests/test_delegate.py @@ -8,8 +8,7 @@ import pytest from Python.Test import DelegateTest, StringDelegate -from ._compat import DictProxyType -from .utils import HelloClass, hello_func, MultipleHandler +from .utils import HelloClass, hello_func, MultipleHandler, DictProxyType def test_delegate_standard_attrs(): diff --git a/src/tests/test_dictionary.py b/src/tests/test_dictionary.py new file mode 100644 index 000000000..98c3cfb20 --- /dev/null +++ b/src/tests/test_dictionary.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +"""Test support for managed dictionaries.""" + +import Python.Test as Test +import System +import pytest + +from ._compat import PY2, UserList, long, range, unichr + + +def test_public_dict(): + """Test public dict.""" + ob = Test.PublicDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_protected_dict(): + """Test protected dict.""" + ob = Test.ProtectedDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_internal_dict(): + """Test internal dict.""" + + with pytest.raises(AttributeError): + ob = Test.InternalDictionaryTest() + _ = ob.items + +def test_private_dict(): + """Test private dict.""" + + with pytest.raises(AttributeError): + ob = Test.PrivateDictionaryTest() + _ = ob.items + +def test_dict_contains(): + """Test dict support for __contains__.""" + + ob = Test.PublicDictionaryTest() + items = ob.items + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) + +def test_dict_abuse(): + """Test dict abuse.""" + _class = Test.PublicDictionaryTest + ob = Test.PublicDictionaryTest() + + with pytest.raises(AttributeError): + del _class.__getitem__ + + with pytest.raises(AttributeError): + del ob.__getitem__ + + with pytest.raises(AttributeError): + del _class.__setitem__ + + with pytest.raises(AttributeError): + del ob.__setitem__ + + with pytest.raises(TypeError): + Test.PublicArrayTest.__getitem__(0, 0) + + with pytest.raises(TypeError): + Test.PublicArrayTest.__setitem__(0, 0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__getitem__'] + desc(0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__setitem__'] + desc(0, 0, 0) + +def test_InheritedDictionary(): + """Test class that inherited from IDictionary.""" + items = Test.InheritedDictionaryTest() + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_InheritedDictionary_contains(): + """Test dict support for __contains__ in class that inherited from IDictionary""" + items = Test.InheritedDictionaryTest() + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) \ No newline at end of file diff --git a/src/tests/test_enum.py b/src/tests/test_enum.py index b31ce4ec5..27fe7e9ef 100644 --- a/src/tests/test_enum.py +++ b/src/tests/test_enum.py @@ -5,7 +5,7 @@ import pytest import Python.Test as Test -from ._compat import DictProxyType, long +from .utils import DictProxyType def test_enum_standard_attrs(): @@ -68,23 +68,23 @@ def test_int_enum(): def test_uint_enum(): """Test uint enum.""" - assert Test.UIntEnum.Zero == long(0) - assert Test.UIntEnum.One == long(1) - assert Test.UIntEnum.Two == long(2) + assert Test.UIntEnum.Zero == 0 + assert Test.UIntEnum.One == 1 + assert Test.UIntEnum.Two == 2 def test_long_enum(): """Test long enum.""" - assert Test.LongEnum.Zero == long(0) - assert Test.LongEnum.One == long(1) - assert Test.LongEnum.Two == long(2) + assert Test.LongEnum.Zero == 0 + assert Test.LongEnum.One == 1 + assert Test.LongEnum.Two == 2 def test_ulong_enum(): """Test ulong enum.""" - assert Test.ULongEnum.Zero == long(0) - assert Test.ULongEnum.One == long(1) - assert Test.ULongEnum.Two == long(2) + assert Test.ULongEnum.Zero == 0 + assert Test.ULongEnum.One == 1 + assert Test.ULongEnum.Two == 2 def test_instantiate_enum_fails(): diff --git a/src/tests/test_event.py b/src/tests/test_event.py index 624b83d44..e9c0ffd8a 100644 --- a/src/tests/test_event.py +++ b/src/tests/test_event.py @@ -5,7 +5,6 @@ import pytest from Python.Test import EventTest, EventArgsTest -from ._compat import range from .utils import (CallableHandler, ClassMethodHandler, GenericHandler, MultipleHandler, StaticMethodHandler, VarCallableHandler, VariableArgsHandler) diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index 08b00d77d..02d3005aa 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -6,8 +6,7 @@ import System import pytest - -from ._compat import PY2, PY3, pickle, text_type +import pickle def test_unified_exception_semantics(): @@ -270,12 +269,6 @@ def test_str_of_exception(): with pytest.raises(FormatException) as cm: Convert.ToDateTime('this will fail') - e = cm.value - # fix for international installation - msg = text_type(e).encode("utf8") - fnd = text_type('System.Convert.ToDateTime').encode("utf8") - assert msg.find(fnd) > -1, msg - def test_python_compat_of_managed_exceptions(): """Test managed exceptions compatible with Python's implementation""" @@ -284,15 +277,11 @@ def test_python_compat_of_managed_exceptions(): e = OverflowException(msg) assert str(e) == msg - assert text_type(e) == msg assert e.args == (msg,) assert isinstance(e.args, tuple) - if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp - elif PY2: - assert repr(e) == "OverflowException(u'Simple message',)" + strexp = "OverflowException('Simple message" + assert repr(e)[:len(strexp)] == strexp def test_exception_is_instance_of_system_object(): @@ -330,7 +319,6 @@ def test_pickling_exceptions(): assert exc.args == loaded.args -@pytest.mark.skipif(PY2, reason="__cause__ isn't implemented in PY2") def test_chained_exceptions(): from Python.Test import ExceptionTest diff --git a/src/tests/test_generic.py b/src/tests/test_generic.py index 69cd4ee7f..9c410271d 100644 --- a/src/tests/test_generic.py +++ b/src/tests/test_generic.py @@ -7,8 +7,6 @@ import System import pytest -from ._compat import PY2, long, unicode, unichr, zip - def assert_generic_wrapper_by_type(ptype, value): """Test Helper""" @@ -137,15 +135,15 @@ def test_python_type_aliasing(): dict_.Add(1, 1) assert dict_[1] == 1 - dict_ = Dictionary[long, long]() + dict_ = Dictionary[int, int]() assert dict_.Count == 0 - dict_.Add(long(1), long(1)) - assert dict_[long(1)] == long(1) + dict_.Add(1, 1) + assert dict_[1] == 1 dict_ = Dictionary[System.Int64, System.Int64]() assert dict_.Count == 0 - dict_.Add(long(1), long(1)) - assert dict_[long(1)] == long(1) + dict_.Add(1, 1) + assert dict_[1] == 1 dict_ = Dictionary[float, float]() assert dict_.Count == 0 @@ -257,19 +255,15 @@ def test_generic_type_binding(): assert_generic_wrapper_by_type(System.Int16, 32767) assert_generic_wrapper_by_type(System.Int32, 2147483647) assert_generic_wrapper_by_type(int, 2147483647) - assert_generic_wrapper_by_type(System.Int64, long(9223372036854775807)) - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - assert_generic_wrapper_by_type(long, long(9223372036854775807)) + assert_generic_wrapper_by_type(System.Int64, 9223372036854775807) assert_generic_wrapper_by_type(System.UInt16, 65000) - assert_generic_wrapper_by_type(System.UInt32, long(4294967295)) - assert_generic_wrapper_by_type(System.UInt64, long(18446744073709551615)) + assert_generic_wrapper_by_type(System.UInt32, 4294967295) + assert_generic_wrapper_by_type(System.UInt64, 18446744073709551615) assert_generic_wrapper_by_type(System.Single, 3.402823e38) assert_generic_wrapper_by_type(System.Double, 1.7976931348623157e308) assert_generic_wrapper_by_type(float, 1.7976931348623157e308) assert_generic_wrapper_by_type(System.Decimal, System.Decimal.One) assert_generic_wrapper_by_type(System.String, "test") - assert_generic_wrapper_by_type(unicode, "test") assert_generic_wrapper_by_type(str, "test") assert_generic_wrapper_by_type(ShortEnum, ShortEnum.Zero) assert_generic_wrapper_by_type(System.Object, InterfaceTest()) @@ -315,19 +309,12 @@ def test_generic_method_type_handling(): assert_generic_method_by_type(System.Int16, 32767) assert_generic_method_by_type(System.Int32, 2147483647) assert_generic_method_by_type(int, 2147483647) - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - assert_generic_method_by_type(System.Int64, long(9223372036854775807)) - assert_generic_method_by_type(long, long(9223372036854775807)) - assert_generic_method_by_type(System.UInt32, long(4294967295)) - assert_generic_method_by_type(System.Int64, long(1844674407370955161)) assert_generic_method_by_type(System.UInt16, 65000) assert_generic_method_by_type(System.Single, 3.402823e38) assert_generic_method_by_type(System.Double, 1.7976931348623157e308) assert_generic_method_by_type(float, 1.7976931348623157e308) assert_generic_method_by_type(System.Decimal, System.Decimal.One) assert_generic_method_by_type(System.String, "test") - assert_generic_method_by_type(unicode, "test") assert_generic_method_by_type(str, "test") assert_generic_method_by_type(ShortEnum, ShortEnum.Zero) assert_generic_method_by_type(System.Object, InterfaceTest()) @@ -355,8 +342,6 @@ def test_correct_overload_selection(): assert Math.Max(atype(value1), atype(value2)) == Math.Max.__overloads__[atype, atype]( atype(value1), atype(value2)) - if PY2 and atype is Int64: - value2 = long(value2) assert Math.Max(atype(value1), value2) == Math.Max.__overloads__[atype, atype]( atype(value1), atype(value2)) @@ -481,7 +466,7 @@ def test_method_overload_selection_with_generic_types(): vtype = GenericWrapper[System.Char] input_ = vtype(65535) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == unichr(65535) + assert value.value == chr(65535) vtype = GenericWrapper[System.Int16] input_ = vtype(32767) @@ -499,16 +484,9 @@ def test_method_overload_selection_with_generic_types(): assert value.value == 2147483647 vtype = GenericWrapper[System.Int64] - input_ = vtype(long(9223372036854775807)) + input_ = vtype(9223372036854775807) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - vtype = GenericWrapper[long] - input_ = vtype(long(9223372036854775807)) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(9223372036854775807) + assert value.value == 9223372036854775807 vtype = GenericWrapper[System.UInt16] input_ = vtype(65000) @@ -516,14 +494,14 @@ def test_method_overload_selection_with_generic_types(): assert value.value == 65000 vtype = GenericWrapper[System.UInt32] - input_ = vtype(long(4294967295)) + input_ = vtype(4294967295) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(4294967295) + assert value.value == 4294967295 vtype = GenericWrapper[System.UInt64] - input_ = vtype(long(18446744073709551615)) + input_ = vtype(18446744073709551615) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value.value == long(18446744073709551615) + assert value.value == 18446744073709551615 vtype = GenericWrapper[System.Single] input_ = vtype(3.402823e38) @@ -628,7 +606,7 @@ def test_overload_selection_with_arrays_of_generic_types(): vtype = System.Array[gtype] input_ = vtype([gtype(65535), gtype(65535)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == unichr(65535) + assert value[0].value == chr(65535) assert value.Length == 2 gtype = GenericWrapper[System.Int16] @@ -654,22 +632,12 @@ def test_overload_selection_with_arrays_of_generic_types(): gtype = GenericWrapper[System.Int64] vtype = System.Array[gtype] - input_ = vtype([gtype(long(9223372036854775807)), - gtype(long(9223372036854775807))]) + input_ = vtype([gtype(9223372036854775807), + gtype(9223372036854775807)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(9223372036854775807) + assert value[0].value == 9223372036854775807 assert value.Length == 2 - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - gtype = GenericWrapper[long] - vtype = System.Array[gtype] - input_ = vtype([gtype(long(9223372036854775807)), - gtype(long(9223372036854775807))]) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(9223372036854775807) - assert value.Length == 2 - gtype = GenericWrapper[System.UInt16] vtype = System.Array[gtype] input_ = vtype([gtype(65000), gtype(65000)]) @@ -679,17 +647,17 @@ def test_overload_selection_with_arrays_of_generic_types(): gtype = GenericWrapper[System.UInt32] vtype = System.Array[gtype] - input_ = vtype([gtype(long(4294967295)), gtype(long(4294967295))]) + input_ = vtype([gtype(4294967295), gtype(4294967295)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(4294967295) + assert value[0].value == 4294967295 assert value.Length == 2 gtype = GenericWrapper[System.UInt64] vtype = System.Array[gtype] - input_ = vtype([gtype(long(18446744073709551615)), - gtype(long(18446744073709551615))]) + input_ = vtype([gtype(18446744073709551615), + gtype(18446744073709551615)]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0].value == long(18446744073709551615) + assert value[0].value == 18446744073709551615 assert value.Length == 2 gtype = GenericWrapper[System.Single] diff --git a/src/tests/test_indexer.py b/src/tests/test_indexer.py index 6f18550d9..6a36a2519 100644 --- a/src/tests/test_indexer.py +++ b/src/tests/test_indexer.py @@ -5,8 +5,6 @@ import Python.Test as Test import pytest -from ._compat import long, unichr - def test_public_indexer(): """Test public indexers.""" @@ -131,8 +129,8 @@ def test_sbyte_indexer(): def test_char_indexer(): """Test char indexers.""" ob = Test.CharIndexerTest() - max_ = unichr(65535) - min_ = unichr(0) + max_ = chr(65535) + min_ = chr(0) assert ob[max_] is None @@ -200,8 +198,8 @@ def test_int32_indexer(): def test_int64_indexer(): """Test Int64 indexers.""" ob = Test.Int64IndexerTest() - max_ = long(9223372036854775807) - min_ = long(-9223372036854775808) + max_ = 9223372036854775807 + min_ = -9223372036854775808 assert ob[max_] is None @@ -246,7 +244,7 @@ def test_uint16_indexer(): def test_uint32_indexer(): """Test UInt32 indexers.""" ob = Test.UInt32IndexerTest() - max_ = long(4294967295) + max_ = 4294967295 min_ = 0 assert ob[max_] is None @@ -269,7 +267,7 @@ def test_uint32_indexer(): def test_uint64_indexer(): """Test UInt64 indexers.""" ob = Test.UInt64IndexerTest() - max_ = long(18446744073709551615) + max_ = 18446744073709551615 min_ = 0 assert ob[max_] is None @@ -435,16 +433,16 @@ def test_object_indexer(): ob[1] = "one" assert ob[1] == "one" - ob[long(1)] = "long" - assert ob[long(1)] == "long" + ob[1] = "long" + assert ob[1] == "long" - with pytest.raises(TypeError): - class Eggs(object): - pass + class Eggs(object): + pass - key = Eggs() - ob = Test.ObjectIndexerTest() - ob[key] = "wrong" + key = Eggs() + ob = Test.ObjectIndexerTest() + ob[key] = "eggs_key" + assert ob[key] == "eggs_key" def test_interface_indexer(): diff --git a/src/tests/test_interface.py b/src/tests/test_interface.py index 997f17264..6b72c1a12 100644 --- a/src/tests/test_interface.py +++ b/src/tests/test_interface.py @@ -5,7 +5,7 @@ import Python.Test as Test import pytest -from ._compat import DictProxyType +from .utils import DictProxyType def test_interface_standard_attrs(): diff --git a/src/tests/test_method.py b/src/tests/test_method.py index ad678611b..d9c033802 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -6,8 +6,6 @@ import pytest from Python.Test import MethodTest -from ._compat import PY2, long, unichr - def test_instance_method_descriptor(): """Test instance method descriptor behavior.""" @@ -257,6 +255,37 @@ def test_non_params_array_in_last_place(): result = MethodTest.TestNonParamsArrayInLastPlace(1, 2, 3) assert result +def test_params_methods_with_no_params(): + """Tests that passing no arguments to a params method + passes an empty array""" + result = MethodTest.TestValueParamsArg() + assert len(result) == 0 + + result = MethodTest.TestOneArgWithParams('Some String') + assert len(result) == 0 + + result = MethodTest.TestTwoArgWithParams('Some String', 'Some Other String') + assert len(result) == 0 + +def test_params_methods_with_non_lists(): + """Tests that passing single parameters to a params + method will convert into an array on the .NET side""" + result = MethodTest.TestOneArgWithParams('Some String', [1, 2, 3, 4]) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', 1, 2, 3, 4) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', [5]) + assert len(result) == 1 + + result = MethodTest.TestOneArgWithParams('Some String', 5) + assert len(result) == 1 + +def test_params_method_with_lists(): + """Tests passing multiple lists to a params object[] method""" + result = MethodTest.TestObjectParamsArg([],[]) + assert len(result) == 2 def test_string_out_params(): """Test use of string out-parameters.""" @@ -473,7 +502,7 @@ def test_explicit_overload_selection(): assert value == u'A' value = MethodTest.Overloaded.__overloads__[System.Char](65535) - assert value == unichr(65535) + assert value == chr(65535) value = MethodTest.Overloaded.__overloads__[System.Int16](32767) assert value == 32767 @@ -485,25 +514,22 @@ def test_explicit_overload_selection(): assert value == 2147483647 value = MethodTest.Overloaded.__overloads__[System.Int64]( - long(9223372036854775807)) - assert value == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - value = MethodTest.Overloaded.__overloads__[long]( - long(9223372036854775807)) - assert value == long(9223372036854775807) + 9223372036854775807 + ) + assert value == 9223372036854775807 value = MethodTest.Overloaded.__overloads__[System.UInt16](65000) assert value == 65000 value = MethodTest.Overloaded.__overloads__[System.UInt32]( - long(4294967295)) - assert value == long(4294967295) + 4294967295 + ) + assert value == 4294967295 value = MethodTest.Overloaded.__overloads__[System.UInt64]( - long(18446744073709551615)) - assert value == long(18446744073709551615) + 18446744073709551615 + ) + assert value == 18446744073709551615 value = MethodTest.Overloaded.__overloads__[System.Single](3.402823e38) assert value == 3.402823e38 @@ -590,8 +616,8 @@ def test_overload_selection_with_array_types(): vtype = Array[System.Char] input_ = vtype([0, 65535]) value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0] == unichr(0) - assert value[1] == unichr(65535) + assert value[0] == chr(0) + assert value[1] == chr(65535) vtype = Array[System.Int16] input_ = vtype([0, 32767]) @@ -612,18 +638,10 @@ def test_overload_selection_with_array_types(): assert value[1] == 2147483647 vtype = Array[System.Int64] - input_ = vtype([0, long(9223372036854775807)]) + input_ = vtype([0, 9223372036854775807]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(9223372036854775807) - - # Python 3 has no explicit long type, use System.Int64 instead - if PY2: - vtype = Array[long] - input_ = vtype([0, long(9223372036854775807)]) - value = MethodTest.Overloaded.__overloads__[vtype](input_) - assert value[0] == 0 - assert value[1] == long(9223372036854775807) + assert value[1] == 9223372036854775807 vtype = Array[System.UInt16] input_ = vtype([0, 65000]) @@ -632,16 +650,16 @@ def test_overload_selection_with_array_types(): assert value[1] == 65000 vtype = Array[System.UInt32] - input_ = vtype([0, long(4294967295)]) + input_ = vtype([0, 4294967295]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(4294967295) + assert value[1] == 4294967295 vtype = Array[System.UInt64] - input_ = vtype([0, long(18446744073709551615)]) + input_ = vtype([0, 18446744073709551615]) value = MethodTest.Overloaded.__overloads__[vtype](input_) assert value[0] == 0 - assert value[1] == long(18446744073709551615) + assert value[1] == 18446744073709551615 vtype = Array[System.Single] input_ = vtype([0.0, 3.402823e38]) @@ -717,7 +735,7 @@ def test_explicit_overload_selection_failure(): _ = MethodTest.Overloaded.__overloads__[str, int, int]("", 1, 1) with pytest.raises(TypeError): - _ = MethodTest.Overloaded.__overloads__[int, long](1) + _ = MethodTest.Overloaded.__overloads__[int, int](1) def test_we_can_bind_to_encoding_get_string(): @@ -777,6 +795,9 @@ def test_no_object_in_param(): res = MethodTest.TestOverloadedNoObject(5) assert res == "Got int" + res = MethodTest.TestOverloadedNoObject(i=7) + assert res == "Got int" + with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") @@ -788,9 +809,15 @@ def test_object_in_param(): res = MethodTest.TestOverloadedObject(5) assert res == "Got int" + res = MethodTest.TestOverloadedObject(i=7) + assert res == "Got int" + res = MethodTest.TestOverloadedObject("test") assert res == "Got object" + res = MethodTest.TestOverloadedObject(o="test") + assert res == "Got object" + def test_object_in_multiparam(): """Test method with object multiparams behaves""" @@ -813,6 +840,42 @@ def test_object_in_multiparam(): res = MethodTest.TestOverloadedObjectTwo(7.24, 7.24) assert res == "Got object-object" + res = MethodTest.TestOverloadedObjectTwo(a=5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(a=5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo("foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo(a=7.24, b=7.24) + assert res == "Got object-object" + + res = MethodTest.TestOverloadedObjectTwo(7.24, b=7.24) + assert res == "Got object-object" + def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" @@ -966,3 +1029,120 @@ def test_getting_overloaded_constructor_binding_does_not_leak_ref_count(): # simple test refCount = sys.getrefcount(PlainOldClass.Overloads[int]) assert refCount == 1 + + +def test_default_params(): + # all positional parameters + res = MethodTest.DefaultParams(1,2,3,4) + assert res == "1234" + + res = MethodTest.DefaultParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.DefaultParams(1, 2) + assert res == "1200" + + res = MethodTest.DefaultParams(1) + assert res == "1000" + + res = MethodTest.DefaultParams(a=2) + assert res == "2000" + + res = MethodTest.DefaultParams(b=3) + assert res == "0300" + + res = MethodTest.DefaultParams(c=4) + assert res == "0040" + + res = MethodTest.DefaultParams(d=7) + assert res == "0007" + + res = MethodTest.DefaultParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.DefaultParams(1, d=7, c=3) + assert res == "1037" + + with pytest.raises(TypeError): + MethodTest.DefaultParams(1,2,3,4,5) + +def test_optional_params(): + res = MethodTest.OptionalParams(1, 2, 3, 4) + assert res == "1234" + + res = MethodTest.OptionalParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.OptionalParams(1, 2) + assert res == "1200" + + res = MethodTest.OptionalParams(1) + assert res == "1000" + + res = MethodTest.OptionalParams(a=2) + assert res == "2000" + + res = MethodTest.OptionalParams(b=3) + assert res == "0300" + + res = MethodTest.OptionalParams(c=4) + assert res == "0040" + + res = MethodTest.OptionalParams(d=7) + assert res == "0007" + + res = MethodTest.OptionalParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.OptionalParams(1, d=7, c=3) + assert res == "1037" + + res = MethodTest.OptionalParams_TestMissing() + assert res == True + + res = MethodTest.OptionalParams_TestMissing(None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a = None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a='hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType() + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(a=None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType('hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType(a='hi') + assert res == False + +def test_optional_and_default_params(): + + res = MethodTest.OptionalAndDefaultParams() + assert res == "0000" + + res = MethodTest.OptionalAndDefaultParams(1) + assert res == "1000" + + res = MethodTest.OptionalAndDefaultParams(1, c=4) + assert res == "1040" + + res = MethodTest.OptionalAndDefaultParams(b=4, c=7) + assert res == "0470" + + res = MethodTest.OptionalAndDefaultParams2() + assert res == "0012" + + res = MethodTest.OptionalAndDefaultParams2(a=1,b=2,c=3,d=4) + assert res == "1234" + + res = MethodTest.OptionalAndDefaultParams2(b=2, c=3) + assert res == "0232" diff --git a/src/tests/test_module.py b/src/tests/test_module.py index 62d79b9ab..2b1a9e4ec 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -10,7 +10,6 @@ import pytest -from ._compat import ClassType, PY2, PY3, range from .utils import is_clr_class, is_clr_module, is_clr_root_module @@ -79,14 +78,9 @@ def test_simple_import(): assert isinstance(sys, types.ModuleType) assert sys.__name__ == 'sys' - if PY3: - import http.client as httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'http.client' - elif PY2: - import httplib - assert isinstance(httplib, types.ModuleType) - assert httplib.__name__ == 'httplib' + import http.client as httplib + assert isinstance(httplib, types.ModuleType) + assert httplib.__name__ == 'http.client' def test_simple_import_with_alias(): @@ -99,14 +93,9 @@ def test_simple_import_with_alias(): assert isinstance(mySys, types.ModuleType) assert mySys.__name__ == 'sys' - if PY3: - import http.client as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'http.client' - elif PY2: - import httplib as myHttplib - assert isinstance(myHttplib, types.ModuleType) - assert myHttplib.__name__ == 'httplib' + import http.client as myHttplib + assert isinstance(myHttplib, types.ModuleType) + assert myHttplib.__name__ == 'http.client' def test_dotted_name_import(): @@ -178,7 +167,7 @@ def test_dotted_name_import_from(): assert pulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM - assert isinstance(PullDOM, ClassType) + assert isinstance(PullDOM, type) assert PullDOM.__name__ == 'PullDOM' @@ -197,7 +186,7 @@ def test_dotted_name_import_from_with_alias(): assert myPulldom.__name__ == 'xml.dom.pulldom' from xml.dom.pulldom import PullDOM as myPullDOM - assert isinstance(myPullDOM, ClassType) + assert isinstance(myPullDOM, type) assert myPullDOM.__name__ == 'PullDOM' diff --git a/src/tests/test_mp_length.py b/src/tests/test_mp_length.py new file mode 100644 index 000000000..c96ac77d1 --- /dev/null +++ b/src/tests/test_mp_length.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +"""Test __len__ for .NET classes implementing ICollection/ICollection.""" + +import System +import pytest +from Python.Test import MpLengthCollectionTest, MpLengthExplicitCollectionTest, MpLengthGenericCollectionTest, MpLengthExplicitGenericCollectionTest + +def test_simple___len__(): + """Test __len__ for simple ICollection implementers""" + import System + import System.Collections.Generic + l = System.Collections.Generic.List[int]() + assert len(l) == 0 + l.Add(5) + l.Add(6) + assert len(l) == 2 + + d = System.Collections.Generic.Dictionary[int, int]() + assert len(d) == 0 + d.Add(4, 5) + assert len(d) == 1 + + a = System.Array[int]([0,1,2,3]) + assert len(a) == 4 + +def test_custom_collection___len__(): + """Test __len__ for custom collection class""" + s = MpLengthCollectionTest() + assert len(s) == 3 + +def test_custom_collection_explicit___len__(): + """Test __len__ for custom collection class that explicitly implements ICollection""" + s = MpLengthExplicitCollectionTest() + assert len(s) == 2 + +def test_custom_generic_collection___len__(): + """Test __len__ for custom generic collection class""" + s = MpLengthGenericCollectionTest[int]() + s.Add(1) + s.Add(2) + assert len(s) == 2 + +def test_custom_generic_collection_explicit___len__(): + """Test __len__ for custom generic collection that explicity implements ICollection""" + s = MpLengthExplicitGenericCollectionTest[int]() + s.Add(1) + s.Add(10) + assert len(s) == 2 diff --git a/src/tests/test_repr.py b/src/tests/test_repr.py new file mode 100644 index 000000000..5131f5d88 --- /dev/null +++ b/src/tests/test_repr.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +"""Test __repr__ output""" + +import System +import pytest +from Python.Test import ReprTest + +def test_basic(): + """Test Point class which implements both ToString and __repr__ without inheritance""" + ob = ReprTest.Point(1,2) + # point implements ToString() and __repr__() + assert ob.__repr__() == "Point(1,2)" + assert str(ob) == "Python.Test.ReprTest+Point: X=1, Y=2" + +def test_system_string(): + """Test system string""" + ob = System.String("hello") + assert str(ob) == "hello" + assert " - @@ -59,6 +58,7 @@ + diff --git a/src/tests/utils.py b/src/tests/utils.py index cacb015ec..b467cae97 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -5,9 +5,7 @@ Refactor utility functions and classes """ -from __future__ import print_function - -from ._compat import PY2, PY3 +DictProxyType = type(object.__dict__) def dprint(msg): @@ -21,11 +19,8 @@ def is_clr_module(ob): def is_clr_root_module(ob): - if PY3: - # in Python 3 the clr module is a normal python module - return ob.__name__ == "clr" - elif PY2: - return type(ob).__name__ == 'CLRModule' + # in Python 3 the clr module is a normal python module + return ob.__name__ == "clr" def is_clr_class(ob): diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..902296229 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -76,6 +76,8 @@ def visit(self, node): self.visit_struct(node) elif isinstance(node, c_ast.Decl): self.visit_decl(node) + elif isinstance(node, c_ast.FuncDecl): + self.visit_funcdecl(node) elif isinstance(node, c_ast.PtrDecl): self.visit_ptrdecl(node) elif isinstance(node, c_ast.IdentifierType): @@ -110,6 +112,9 @@ def visit_struct(self, struct): def visit_decl(self, decl): self.visit(decl.type) + def visit_funcdecl(self, funcdecl): + self.visit(funcdecl.type) + def visit_ptrdecl(self, ptrdecl): self.__ptr_decl_depth += 1 self.visit(ptrdecl.type) @@ -169,6 +174,8 @@ def preprocess_python_headers(): include_py = sysconfig.get_config_var("INCLUDEPY") include_dirs.append(include_py) + include_args = [c for p in include_dirs for c in ["-I", p]] + defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", @@ -177,6 +184,13 @@ def preprocess_python_headers(): "-D", "_POSIX_THREADS" ] + if os.name == 'nt': + defines.extend([ + "-D", "__inline=inline", + "-D", "__ptr32=", + "-D", "__declspec(x)=", + ]) + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) @@ -186,7 +200,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = [] @@ -216,7 +230,7 @@ def gen_interop_code(members): defines_str = " && ".join(defines) class_definition = """ // Auto-generated by %s. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if %s diff --git a/tools/vswhere/vswhere.exe b/tools/vswhere/vswhere.exe index 3eb2df009..582e82868 100644 Binary files a/tools/vswhere/vswhere.exe and b/tools/vswhere/vswhere.exe differ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 3c9be5c63..000000000 --- a/tox.ini +++ /dev/null @@ -1,27 +0,0 @@ -[tox] -skip_missing_interpreters=True -envlist = - py27 - py33 - py34 - py35 - py36 - py37 - check - -[testenv] -deps = - pytest -commands = - {posargs:py.test} - -[testenv:check] -ignore_errors=True -skip_install=True -basepython=python3.5 -deps = - check-manifest - flake8 -commands = - flake8 src setup.py - python setup.py check --strict --metadata