From 4c4c70136ffb57bc7f4f3ebfb9f1ec46c0b81408 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 10 Jul 2020 15:31:28 -0700 Subject: [PATCH 01/60] Add new type object for union types --- Include/Python.h | 1 + Include/unionobject.h | 13 ++++++++++++ Objects/unionobject.c | 47 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 Include/unionobject.h create mode 100644 Objects/unionobject.c diff --git a/Include/Python.h b/Include/Python.h index 57f71d41d8d477..3bfd491f6106f4 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -123,6 +123,7 @@ #include "genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "unionobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/unionobject.h b/Include/unionobject.h new file mode 100644 index 00000000000000..9d8125f2ea83f7 --- /dev/null +++ b/Include/unionobject.h @@ -0,0 +1,13 @@ +#ifndef Py_UNIONTYPEOBJECT_H +#define Py_UNIONTYPEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_FUNC(PyObject *) Py_Union(PyObject *, PyObject *); +PyAPI_DATA(PyTypeObject) Py_UnionType; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_UNIONTYPEOBJECT_H */ diff --git a/Objects/unionobject.c b/Objects/unionobject.c new file mode 100644 index 00000000000000..efb522b5a4e330 --- /dev/null +++ b/Objects/unionobject.c @@ -0,0 +1,47 @@ +// types.Union -- used to represent e.g. Union[int, str], int | str +#include "Python.h" +#include "pycore_object.h" +#include "structmember.h" + +typedef struct { + PyObject_HEAD; + PyObject *origin; + PyObject *args; +} unionobject; + +static void +unionobject_dealloc(PyObject *self) +{ + unionobject *alias = (unionobject *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(alias->origin); + Py_XDECREF(alias->args); + self->ob_type->tp_free(self); +} + +PyTypeObject Py_UnionType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "types.Union", + .tp_doc = "Represent a PEP 604 union type\n" + "\n" + "E.g. for int | str", + .tp_basicsize = sizeof(unionobject), + .tp_dealloc = unionobject_dealloc +}; + +PyObject * +Py_Union(PyObject *origin, PyObject *args) +{ + unionobject *alias = PyObject_GC_New(unionobject, &Py_UnionType); + if (alias == NULL) { + Py_DECREF(args); + return NULL; + } + + Py_INCREF(origin); + alias->origin = origin; + alias->args = args; + _PyObject_GC_TRACK(alias); + return (PyObject *)alias; +} From 07b5c2934c63dead9641d83b749bdb4542bfd29b Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 10 Jul 2020 16:23:32 -0700 Subject: [PATCH 02/60] Add test. --- Lib/test/test_isinstance.py | 16 +++++++ Lib/test/test_types.py | 88 ++++++++++++++++++++++++++++++++++++- Lib/test/test_typing.py | 10 ++--- Lib/typing.py | 14 ++++++ Makefile.pre.in | 1 + Objects/abstract.c | 40 ++++++++++++++++- Objects/typeobject.c | 43 +++++++++++++++++- PCbuild/pythoncore.vcxproj | 1 + 8 files changed, 204 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 53639e984e48a7..464242904ab1eb 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -4,6 +4,7 @@ import unittest import sys +import typing @@ -208,6 +209,15 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Super)) self.assertEqual(False, isinstance(AbstractChild(), Child)) + def test_isinstance_with_or_union(self): + self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) + self.assertEqual(False, isinstance(None, str | int)) + self.assertEqual(True, isinstance(3, str | int)) + self.assertEqual(True, isinstance("", str | int)) + self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) + self.assertEqual(True, isinstance(2, typing.List | int)) + self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) + def test_subclass_normal(self): # normal classes self.assertEqual(True, issubclass(Super, Super)) @@ -217,6 +227,8 @@ def test_subclass_normal(self): self.assertEqual(True, issubclass(Child, Child)) self.assertEqual(True, issubclass(Child, Super)) self.assertEqual(False, issubclass(Child, AbstractSuper)) + self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) + self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) def test_subclass_abstract(self): # abstract classes @@ -251,6 +263,10 @@ def test_isinstance_recursion_limit(self): # blown self.assertRaises(RecursionError, blowstack, isinstance, '', str) + # def test_subclass_with_union(self): + # self.assertEqual(True, issubclass(int, int | float | int)) + # self.assertEqual(True, issubclass(str, str | Child | str)) + def test_issubclass_refcount_handling(self): # bpo-39382: abstract_issubclass() didn't hold item reference while # peeking in the bases tuple, in the single inheritance case. diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 49dc5bf40e3ed8..acf50a885d4f47 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2,6 +2,7 @@ from test.support import run_with_locale import collections.abc +from collections import namedtuple import inspect import pickle import locale @@ -9,6 +10,7 @@ import types import unittest.mock import weakref +import typing class TypesTests(unittest.TestCase): @@ -598,7 +600,91 @@ def test_method_descriptor_types(self): self.assertIsInstance(int.from_bytes, types.BuiltinMethodType) self.assertIsInstance(int.__new__, types.BuiltinMethodType) - + def test_or_types_operator(self): + self.assertEqual(int | str, typing.Union[int, str]) + self.assertEqual(int | None, typing.Union[int, None]) + self.assertEqual(None | int, typing.Union[int, None]) + self.assertEqual(int | str | list, typing.Union[int, str, list]) + self.assertEqual(int | (str | list), typing.Union[int, str, list]) + self.assertEqual(typing.List | typing.Tuple, typing.Union[typing.List, typing.Tuple]) + self.assertEqual(typing.List[int] | typing.Tuple[int], typing.Union[typing.List[int], typing.Tuple[int]]) + self.assertEqual(typing.List[int] | None, typing.Union[typing.List[int], None]) + self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) + self.assertEqual( + BaseException | \ + bool | \ + bytes | \ + complex | \ + float | \ + int | \ + list | \ + map | \ + set, + typing.Union[ + BaseException, + bool, + bytes, + complex, + float, + int, + list, + map, + set, + ]) + with self.assertRaises(TypeError): + int | 3 + with self.assertRaises(TypeError): + 3 | int + + def test_or_type_operator_with_TypeVar(self): + TV = typing.TypeVar('T') + assert TV | str == typing.Union[TV, str] + assert str | TV == typing.Union[str, TV] + + def test_or_type_operator_with_forward(self): + T = typing.TypeVar('T') + ForwardAfter = T | 'Forward' + ForwardBefore = 'Forward' | T + def forward_after(x: ForwardAfter[int]) -> None: ... + def forward_before(x: ForwardBefore[int]) -> None: ... + assert typing.get_args(typing.get_type_hints(forward_after)['x']) == (int, Forward) + assert typing.get_args(typing.get_type_hints(forward_before)['x']) == (int, Forward) + + def test_or_type_operator_with_Protocol(self): + class Proto(typing.Protocol): + def meth(self) -> int: + ... + assert Proto | str == typing.Union[Proto, str] + + def test_or_type_operator_with_Alias(self): + assert list | str == typing.Union[list, str] + assert typing.List | str == typing.Union[typing.List, str] + + def test_or_type_operator_with_NamedTuple(self): + NT=namedtuple('A', ['B', 'C', 'D']) + assert NT | str == typing.Union[NT,str] + + def test_or_type_operator_with_TypedDict(self): + class Point2D(typing.TypedDict): + x: int + y: int + label: str + assert Point2D | str == typing.Union[Point2D, str] + + def test_or_type_operator_with_NewType(self): + UserId = typing.NewType('UserId', int) + assert UserId | str == typing.Union[UserId, str] + + def test_or_type_operator_with_IO(self): + assert typing.IO | str == typing.Union[typing.IO, str] + +class Forward: ... +# T = typing.TypeVar('T') +# ForwardAfter = T | 'Forward' +# ForwardBefore = 'Forward' | T +# def forward_after(x: ForwardAfter[int]) -> None: ... +# def forward_before(x: ForwardBefore[int]) -> None: ... +# class Forward: ... class MappingProxyTests(unittest.TestCase): mappingproxy = types.MappingProxyType diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f429e883b59538..e52ce095345fca 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -244,8 +244,8 @@ def test_subclass_error(self): issubclass(int, Union) with self.assertRaises(TypeError): issubclass(Union, int) - with self.assertRaises(TypeError): - issubclass(int, Union[int, str]) + # with self.assertRaises(TypeError): + # issubclass(int, Union[int, str]) with self.assertRaises(TypeError): issubclass(Union[int, str], int) @@ -347,9 +347,9 @@ def test_empty(self): with self.assertRaises(TypeError): Union[()] - def test_union_instance_type_error(self): - with self.assertRaises(TypeError): - isinstance(42, Union[int, str]) + # def test_union_instance_type_error(self): + # with self.assertRaises(TypeError): + # isinstance(42, Union[int, str]) def test_no_eval_union(self): u = Union[int, str] diff --git a/Lib/typing.py b/Lib/typing.py index f94996daebd6ed..d6f0e75f68b685 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -578,6 +578,13 @@ def __init__(self, name, *constraints, bound=None, if def_mod != 'typing': self.__module__ = def_mod + def __or__(self,right): + return Union[self,right] + def __ror__(self,right): + return Union[self,right] + def __invert__(self): + return Union[self,None] + def __repr__(self): if self.__covariant__: prefix = '+' @@ -685,6 +692,13 @@ def __eq__(self, other): def __hash__(self): return hash((self.__origin__, self.__args__)) + def __or__(self,right): + return Union[self,right] + def __ror__(self,right): + return Union[self,right] + def __invert__(self): + return Union[self,None] + @_tp_cache def __getitem__(self, params): if self.__origin__ in (Generic, Protocol): diff --git a/Makefile.pre.in b/Makefile.pre.in index 3428b9842a5a0f..fc204f53b5e122 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -429,6 +429,7 @@ OBJECT_OBJS= \ Objects/typeobject.o \ Objects/unicodeobject.o \ Objects/unicodectype.o \ + Objects/unionobject.o \ Objects/weakrefobject.o ########################################################################## diff --git a/Objects/abstract.c b/Objects/abstract.c index b9e7111299e2fe..7fbbb75edb6c91 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2469,6 +2469,40 @@ abstract_issubclass(PyObject *derived, PyObject *cls) } } +static PyObject* +union_to_tuple(PyObject* cls) { + //printf("union_to_tuple"); + if (!strcmp(Py_TYPE(cls)->tp_name,"_GenericAlias")) { + PyObject* origin = PyObject_GetAttrString(cls, "__origin__"); + //printf("origin = %p\n",origin); + if (origin == NULL) { + printf("origin == NULL, return cls"); + return cls; + } + //printf("X origin = %s\n",Py_TYPE(cls)->tp_name); + if (PyObject_HasAttrString(origin, "_name")) { + PyObject* name = PyObject_GetAttrString(origin, "_name"); + if (name==NULL) { + printf("_name = NULL\n"); + Py_DECREF(origin); + return cls; + } + //printf("name = %s\n",Py_TYPE(name)->tp_name); + const char* data = (char*)PyUnicode_1BYTE_DATA(name); + //printf("DATA=%s\n",data); + if (data != NULL && !strcmp(data,"Union")) { + PyObject* new_cls = PyObject_GetAttrString(cls, "__args__"); + if (new_cls != NULL) { + cls = new_cls; + } + } + Py_DECREF(name); + } + Py_DECREF(origin); + } + return cls; +} + static int check_class(PyObject *cls, const char *error) { @@ -2511,7 +2545,7 @@ object_isinstance(PyObject *inst, PyObject *cls) } else { if (!check_class(cls, - "isinstance() arg 2 must be a type or tuple of types")) + "isinstance() arg 2 must be a type, a tuple of types or union")) return -1; retval = _PyObject_LookupAttrId(inst, &PyId___class__, &icls); if (icls != NULL) { @@ -2537,7 +2571,7 @@ object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls if (PyType_CheckExact(cls)) { return object_isinstance(inst, cls); } - + cls = union_to_tuple(cls); if (PyTuple_Check(cls)) { /* Not a general sequence -- that opens up the road to recursion and stack overflow. */ @@ -2626,6 +2660,8 @@ object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls) return recursive_issubclass(derived, cls); } + cls = union_to_tuple(cls); + if (PyTuple_Check(cls)) { if (_Py_EnterRecursiveCall(tstate, " in __subclasscheck__")) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f0e349ecd2bb92..1ef92a3dffd0b1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3741,6 +3741,47 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } +static PyObject * +type_or(PyTypeObject* self, PyObject* param) { + PyObject* typing=PyImport_ImportModule("typing"); + PyTypeObject* genericAlias = (PyTypeObject*)PyObject_GetAttrString(typing,"_GenericAlias"); + PyTypeObject* typeVar = (PyTypeObject*)PyObject_GetAttrString(typing,"TypeVar"); + + // Check param is a PyType or GenericAlias + if ((param == NULL) || + ( + (param != Py_None) && + ! PyType_IsSubtype(genericAlias, Py_TYPE(param)) && + ! PyType_IsSubtype(typeVar, Py_TYPE(param)) && + (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) + ) + ) { + PyErr_SetString(PyExc_TypeError, "'type' expected"); + Py_DECREF(typeVar); + Py_DECREF(genericAlias); + Py_DECREF(typing); + return NULL; + } + + // 1. Create a tuple with types + PyObject *tuple=PyTuple_Pack(2,self, param); + // 2. Create Union with tuple + PyObject* unionType = PyObject_GetAttrString(typing,"Union"); + PyObject *newUnion=PyObject_GetItem(unionType, tuple); + // 3. Clean memory + Py_DECREF(typeVar); + Py_DECREF(genericAlias); + Py_DECREF(typing); + Py_DECREF(unionType); + Py_DECREF(tuple); + // 4. Return instance + return newUnion; +} + +static PyNumberMethods type_as_number = { + .nb_or = (binaryfunc)type_or, // Add __or__ function +}; + PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ @@ -3752,7 +3793,7 @@ PyTypeObject PyType_Type = { 0, /* tp_setattr */ 0, /* tp_as_async */ (reprfunc)type_repr, /* tp_repr */ - 0, /* tp_as_number */ + &type_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index db26e38911bc05..104956d265cd17 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -412,6 +412,7 @@ + From 32c207b5e3bb291f634623be23ce2c0e94b057d4 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 15 Jul 2020 10:05:15 -0700 Subject: [PATCH 03/60] Add basic tp_* fields to union object --- Include/unionobject.h | 2 +- Objects/abstract.c | 2 ++ Objects/typeobject.c | 13 ++++++++++--- Objects/unionobject.c | 43 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Include/unionobject.h b/Include/unionobject.h index 9d8125f2ea83f7..d72c3e88e0ca67 100644 --- a/Include/unionobject.h +++ b/Include/unionobject.h @@ -4,7 +4,7 @@ extern "C" { #endif -PyAPI_FUNC(PyObject *) Py_Union(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) Py_Union(PyObject *); PyAPI_DATA(PyTypeObject) Py_UnionType; #ifdef __cplusplus diff --git a/Objects/abstract.c b/Objects/abstract.c index 7fbbb75edb6c91..09de6d07c19efb 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2571,7 +2571,9 @@ object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls if (PyType_CheckExact(cls)) { return object_isinstance(inst, cls); } + cls = union_to_tuple(cls); + if (PyTuple_Check(cls)) { /* Not a general sequence -- that opens up the road to recursion and stack overflow. */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1ef92a3dffd0b1..6eac8b92954460 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3741,12 +3741,16 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } + static PyObject * type_or(PyTypeObject* self, PyObject* param) { + // PyObject_Print(param, stdout, 0); + // PyObject_Print(self, stdout, 0); + // printf("\n"); PyObject* typing=PyImport_ImportModule("typing"); PyTypeObject* genericAlias = (PyTypeObject*)PyObject_GetAttrString(typing,"_GenericAlias"); PyTypeObject* typeVar = (PyTypeObject*)PyObject_GetAttrString(typing,"TypeVar"); - + // printf("Y: %d", Py_Type(*param)); // Check param is a PyType or GenericAlias if ((param == NULL) || ( @@ -3762,9 +3766,11 @@ type_or(PyTypeObject* self, PyObject* param) { Py_DECREF(typing); return NULL; } - // 1. Create a tuple with types - PyObject *tuple=PyTuple_Pack(2,self, param); + PyObject *tuple=PyTuple_Pack(2, self, param); + PyObject *newUnionType=Py_Union(tuple); + // printf("%s", newUnionType.args); + // PyObject_Print(newUnionType, stdout, 0); // 2. Create Union with tuple PyObject* unionType = PyObject_GetAttrString(typing,"Union"); PyObject *newUnion=PyObject_GetItem(unionType, tuple); @@ -3774,6 +3780,7 @@ type_or(PyTypeObject* self, PyObject* param) { Py_DECREF(typing); Py_DECREF(unionType); Py_DECREF(tuple); + Py_DECREF(newUnionType); // 4. Return instance return newUnion; } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index efb522b5a4e330..24bb944ab5c732 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -5,7 +5,6 @@ typedef struct { PyObject_HEAD; - PyObject *origin; PyObject *args; } unionobject; @@ -15,33 +14,61 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); - Py_XDECREF(alias->origin); + // Py_XDECREF(alias->origin); Py_XDECREF(alias->args); self->ob_type->tp_free(self); } + + +// static PyObject * +// unionobject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +// { +// printf("Hello."); +// PyObject *origin = PyTuple_GET_ITEM(args, 0); +// PyObject *arguments = PyTuple_GET_ITEM(args, 1); +// return Py_Union(origin, arguments); +// } + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "types.Union", + .tp_name = "typing.Union", .tp_doc = "Represent a PEP 604 union type\n" "\n" "E.g. for int | str", .tp_basicsize = sizeof(unionobject), - .tp_dealloc = unionobject_dealloc + .tp_dealloc = unionobject_dealloc, + .tp_alloc = PyType_GenericAlloc, + .tp_free = PyObject_GC_Del, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + // .tp_new = unionobject_new }; PyObject * -Py_Union(PyObject *origin, PyObject *args) +Py_Union(PyObject *args) { + if (!PyTuple_Check(args)) { + args = PyTuple_Pack(1, args); + if (args == NULL) { + return NULL; + } + } + else { + Py_INCREF(args); + } + + printf("Here, creating a new union"); unionobject *alias = PyObject_GC_New(unionobject, &Py_UnionType); if (alias == NULL) { Py_DECREF(args); return NULL; } - Py_INCREF(origin); - alias->origin = origin; + PyObject_Print(args, stdout, 0); + // alias->origin = NULL; + // alias->params = NULL; alias->args = args; _PyObject_GC_TRACK(alias); - return (PyObject *)alias; + return (PyObject *) alias; + // return alias; } From 50b33c22dc3f48e081bf8fdf2fea7f7ab12bef2d Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 15 Jul 2020 11:43:25 -0700 Subject: [PATCH 04/60] Squash this commit. --- Objects/typeobject.c | 62 ++++++++++++++++++++++++++----------------- Objects/unionobject.c | 16 ----------- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 6eac8b92954460..8255f8bd03247a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3741,48 +3741,60 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } +static int +is_typevar(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (strcmp(type->tp_name, "TypeVar") != 0) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); + if (module == NULL) { + return -1; + } + int res = PyUnicode_Check(module) + && _PyUnicode_EqualToASCIIString(module, "typing"); + Py_DECREF(module); + return res; +} + +static int +is_genericalias(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (strcmp(type->tp_name, "GenericAlias") != 0) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); + if (module == NULL) { + return -1; + } + int res = PyUnicode_Check(module) + && _PyUnicode_EqualToASCIIString(module, "typing"); + Py_DECREF(module); + return res; +} static PyObject * type_or(PyTypeObject* self, PyObject* param) { - // PyObject_Print(param, stdout, 0); - // PyObject_Print(self, stdout, 0); - // printf("\n"); - PyObject* typing=PyImport_ImportModule("typing"); - PyTypeObject* genericAlias = (PyTypeObject*)PyObject_GetAttrString(typing,"_GenericAlias"); - PyTypeObject* typeVar = (PyTypeObject*)PyObject_GetAttrString(typing,"TypeVar"); - // printf("Y: %d", Py_Type(*param)); // Check param is a PyType or GenericAlias if ((param == NULL) || ( (param != Py_None) && - ! PyType_IsSubtype(genericAlias, Py_TYPE(param)) && - ! PyType_IsSubtype(typeVar, Py_TYPE(param)) && + ! is_genericalias(param) && + ! is_typevar(param) && (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) ) ) { PyErr_SetString(PyExc_TypeError, "'type' expected"); - Py_DECREF(typeVar); - Py_DECREF(genericAlias); - Py_DECREF(typing); + // Py_DECREF(typing); return NULL; } // 1. Create a tuple with types PyObject *tuple=PyTuple_Pack(2, self, param); PyObject *newUnionType=Py_Union(tuple); - // printf("%s", newUnionType.args); - // PyObject_Print(newUnionType, stdout, 0); - // 2. Create Union with tuple - PyObject* unionType = PyObject_GetAttrString(typing,"Union"); - PyObject *newUnion=PyObject_GetItem(unionType, tuple); - // 3. Clean memory - Py_DECREF(typeVar); - Py_DECREF(genericAlias); - Py_DECREF(typing); - Py_DECREF(unionType); Py_DECREF(tuple); - Py_DECREF(newUnionType); - // 4. Return instance - return newUnion; + return newUnionType; } static PyNumberMethods type_as_number = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 24bb944ab5c732..338dbe7ae2cbaa 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -20,16 +20,6 @@ unionobject_dealloc(PyObject *self) } - -// static PyObject * -// unionobject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -// { -// printf("Hello."); -// PyObject *origin = PyTuple_GET_ITEM(args, 0); -// PyObject *arguments = PyTuple_GET_ITEM(args, 1); -// return Py_Union(origin, arguments); -// } - PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "typing.Union", @@ -41,7 +31,6 @@ PyTypeObject Py_UnionType = { .tp_alloc = PyType_GenericAlloc, .tp_free = PyObject_GC_Del, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - // .tp_new = unionobject_new }; PyObject * @@ -57,18 +46,13 @@ Py_Union(PyObject *args) Py_INCREF(args); } - printf("Here, creating a new union"); unionobject *alias = PyObject_GC_New(unionobject, &Py_UnionType); if (alias == NULL) { Py_DECREF(args); return NULL; } - PyObject_Print(args, stdout, 0); - // alias->origin = NULL; - // alias->params = NULL; alias->args = args; _PyObject_GC_TRACK(alias); return (PyObject *) alias; - // return alias; } From 8901145af8b113ece816d1ca0c6d48049dded7a8 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 15 Jul 2020 12:24:54 -0700 Subject: [PATCH 05/60] Squash this commit. --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8255f8bd03247a..bd5201309f873e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3778,6 +3778,7 @@ is_genericalias(PyObject *obj) static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias + printf("Type_or \n"); if ((param == NULL) || ( (param != Py_None) && @@ -5646,7 +5647,6 @@ PyType_Ready(PyTypeObject *type) add_subclass((PyTypeObject *)b, type) < 0) goto error; } - /* All done -- set the ready flag */ type->tp_flags = (type->tp_flags & ~Py_TPFLAGS_READYING) | Py_TPFLAGS_READY; From 6450321d9b0164b201b9f28bd7e6eddee109d81a Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 15 Jul 2020 12:59:21 -0700 Subject: [PATCH 06/60] Add union_traverse method. --- Objects/unionobject.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 338dbe7ae2cbaa..4de05a9a00ee18 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -20,6 +20,26 @@ unionobject_dealloc(PyObject *self) } +static Py_hash_t +union_hash(PyObject *self) +{ + unionobject *alias = (unionobject *)self; + Py_hash_t h1 = PyObject_Hash(alias->args); + if (h1 == -1) { + return -1; + } + return h1; +} + +static int +union_traverse(PyObject *self, visitproc visit, void *arg) +{ + unionobject *alias = (unionobject *)self; + Py_VISIT(alias->args); + return 0; +} + + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "typing.Union", @@ -31,6 +51,9 @@ PyTypeObject Py_UnionType = { .tp_alloc = PyType_GenericAlloc, .tp_free = PyObject_GC_Del, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_hash = union_hash, + .tp_traverse = union_traverse, + }; PyObject * From 8a09a9211bd6aae82c719c67370cdcfa229579b3 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 16 Jul 2020 11:51:07 -0700 Subject: [PATCH 07/60] Handle Py_None in union. --- Lib/test/test_types.py | 2 +- Objects/abstract.c | 11 ++++-- Objects/typeobject.c | 52 ++++++++++++++++++--------- Objects/unionobject.c | 81 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 124 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index acf50a885d4f47..59bc40dd64d419 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -603,7 +603,7 @@ def test_method_descriptor_types(self): def test_or_types_operator(self): self.assertEqual(int | str, typing.Union[int, str]) self.assertEqual(int | None, typing.Union[int, None]) - self.assertEqual(None | int, typing.Union[int, None]) + # self.assertEqual(None | int, typing.Union[int, None]) self.assertEqual(int | str | list, typing.Union[int, str, list]) self.assertEqual(int | (str | list), typing.Union[int, str, list]) self.assertEqual(typing.List | typing.Tuple, typing.Union[typing.List, typing.Tuple]) diff --git a/Objects/abstract.c b/Objects/abstract.c index 09de6d07c19efb..745750c4b272f3 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2471,8 +2471,12 @@ abstract_issubclass(PyObject *derived, PyObject *cls) static PyObject* union_to_tuple(PyObject* cls) { - //printf("union_to_tuple"); - if (!strcmp(Py_TYPE(cls)->tp_name,"_GenericAlias")) { + // if (!strcmp(Py_TYPE(cls)->tp_name, "typing.Union")) { + // PyObject* args = PyObject_GetAttrString(cls, "__args__"); + // PyObject_Print(args, stdout, 0); + // // return args; + // } + if (!strcmp(Py_TYPE(cls)->tp_name, "_GenericAlias")) { PyObject* origin = PyObject_GetAttrString(cls, "__origin__"); //printf("origin = %p\n",origin); if (origin == NULL) { @@ -2493,6 +2497,7 @@ union_to_tuple(PyObject* cls) { if (data != NULL && !strcmp(data,"Union")) { PyObject* new_cls = PyObject_GetAttrString(cls, "__args__"); if (new_cls != NULL) { + printf("A down here."); cls = new_cls; } } @@ -2525,7 +2530,6 @@ object_isinstance(PyObject *inst, PyObject *cls) PyObject *icls; int retval; _Py_IDENTIFIER(__class__); - if (PyType_Check(cls)) { retval = PyObject_TypeCheck(inst, (PyTypeObject *)cls); if (retval == 0) { @@ -2536,6 +2540,7 @@ object_isinstance(PyObject *inst, PyObject *cls) (PyTypeObject *)icls, (PyTypeObject *)cls); } + else { retval = 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bd5201309f873e..a0f2f9272d6276 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3742,10 +3742,10 @@ type_is_gc(PyTypeObject *type) } static int -is_typevar(PyObject *obj) +is_typing_name(PyObject *obj, char *name) { PyTypeObject *type = Py_TYPE(obj); - if (strcmp(type->tp_name, "TypeVar") != 0) { + if (strcmp(type->tp_name, name) != 0) { return 0; } PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); @@ -3758,27 +3758,23 @@ is_typevar(PyObject *obj) return res; } +// TODO: MM: Move these? +static int +is_typevar(PyObject *obj) +{ + return is_typing_name(obj, "TypeVar"); +} + static int is_genericalias(PyObject *obj) { - PyTypeObject *type = Py_TYPE(obj); - if (strcmp(type->tp_name, "GenericAlias") != 0) { - return 0; - } - PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); - if (module == NULL) { - return -1; - } - int res = PyUnicode_Check(module) - && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - return res; + return is_typing_name(obj, "GenericAlias"); } + static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - printf("Type_or \n"); if ((param == NULL) || ( (param != Py_None) && @@ -3787,12 +3783,34 @@ type_or(PyTypeObject* self, PyObject* param) { (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) ) ) { + PyObject_Print(param, stdout, 0); + PyObject_Print(self, stdout, 0); + printf("%d", PyObject_IsInstance(param, (PyObject *) &PyType_Type)); PyErr_SetString(PyExc_TypeError, "'type' expected"); - // Py_DECREF(typing); return NULL; } + PyObject *tuple; // 1. Create a tuple with types - PyObject *tuple=PyTuple_Pack(2, self, param); + if (PyObject_IsInstance((PyObject *)self, (PyObject *) &Py_UnionType) && PyObject_IsInstance((PyObject *)param, (PyObject *) &_PyNone_Type) != 1) { + PyObject* existingArgs = PyObject_GetAttrString((PyObject *)self, "__args__"); + int tuple_size = PyTuple_GET_SIZE(existingArgs); + tuple = PyTuple_New(tuple_size + 1); + for (Py_ssize_t iarg = 0; iarg < tuple_size; iarg++) { + PyObject* arg = PyTuple_GET_ITEM(existingArgs, iarg); + PyTuple_SET_ITEM(tuple, iarg, arg); + } + PyTuple_SET_ITEM(tuple, tuple_size, param); + Py_DECREF(existingArgs); + + } else if (param == Py_None) { + PyTypeObject *type = Py_TYPE(param); + tuple=PyTuple_Pack(2, self, type); + } else if (self == Py_None) { + PyTypeObject *type = Py_TYPE(self); + tuple=PyTuple_Pack(2, type, param); + } else { + tuple=PyTuple_Pack(2, self, param); + } PyObject *newUnionType=Py_Union(tuple); Py_DECREF(tuple); return newUnionType; diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 4de05a9a00ee18..9721e1fb052680 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -39,6 +39,82 @@ union_traverse(PyObject *self, visitproc visit, void *arg) return 0; } +static PyMemberDef union_members[] = { + {"__args__", T_OBJECT, offsetof(unionobject, args), READONLY}, + {0} +}; + + +// TODO; MM: Redo the implementation of this method; +static PyObject * +union_getattro(PyObject *self, PyObject *name) +{ + // printf("union_getattro"); + unionobject *alias = (unionobject *) self; + // PyObject_Print(name, stdout, 0); + // printf("tp_name: (%s)\n", Py_TYPE(name)->tp_name); + return alias->args; + // return PyObject_GenericGetAttr(alias, name); +} + +// TODO: MM: Implement this for isinstance checks +static PyObject * +union_instancecheck(PyObject *self, PyObject *instance) +{ + unionobject *alias = (unionobject *) self; + Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + int retval; + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (PyType_Check(arg)) { + retval = PyObject_IsInstance(instance, arg); + } + } + return self; +} + + +static int +is_typing_name(PyObject *obj, char *name) +{ + PyTypeObject *type = Py_TYPE(obj); + if (strcmp(type->tp_name, name) != 0) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); + if (module == NULL) { + return -1; + } + int res = PyUnicode_Check(module) + && _PyUnicode_EqualToASCIIString(module, "typing"); + Py_DECREF(module); + return res; +} + +static PyMethodDef union_methods[] = { + {"__instancecheck__", union_instancecheck, METH_O}, + {0} +}; + +static PyObject * +union_richcompare(PyObject *a, PyObject *b, int op) +{ + unionobject *aa = (unionobject *)a; + if (is_typing_name(b, "_UnionGenericAlias")) { + PyObject* b_args = PyObject_GetAttrString(b, "__args__"); + printf("\n"); + PyObject_Print(b_args, stdout, 0); + PyObject_Print(aa->args, stdout, 0); + return PyObject_RichCompare(aa->args, b_args, Py_EQ); + } + PyTypeObject *type = Py_TYPE(b); + if (strcmp(type->tp_name, "typing.Union") != 0) { + unionobject *bb = (unionobject *)a; + return PyObject_RichCompare(aa->args, bb->args, Py_EQ); + } + Py_RETURN_FALSE; +} + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -53,7 +129,10 @@ PyTypeObject Py_UnionType = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_hash = union_hash, .tp_traverse = union_traverse, - + .tp_getattro = union_getattro, + .tp_members = union_members, + .tp_methods = union_methods, + .tp_richcompare = union_richcompare, }; PyObject * From e3c80e946aef90c919e94605e92c669297b4f85c Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 16 Jul 2020 11:55:31 -0700 Subject: [PATCH 08/60] Clean up syntax with Py_None --- Objects/typeobject.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a0f2f9272d6276..276f21c5b2f0a3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3802,14 +3802,10 @@ type_or(PyTypeObject* self, PyObject* param) { PyTuple_SET_ITEM(tuple, tuple_size, param); Py_DECREF(existingArgs); - } else if (param == Py_None) { - PyTypeObject *type = Py_TYPE(param); - tuple=PyTuple_Pack(2, self, type); - } else if (self == Py_None) { - PyTypeObject *type = Py_TYPE(self); - tuple=PyTuple_Pack(2, type, param); } else { - tuple=PyTuple_Pack(2, self, param); + PyTypeObject *param_type = param == Py_None ? Py_TYPE(param) : param; + PyTypeObject *self_type = self == Py_None ? Py_TYPE(self) : self; + tuple=PyTuple_Pack(2, self_type, param_type); } PyObject *newUnionType=Py_Union(tuple); Py_DECREF(tuple); From 165c32376a62fc6f66fdd878596b4ca6f8546afb Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 16 Jul 2020 14:33:38 -0700 Subject: [PATCH 09/60] Update getattro method. --- Objects/unionobject.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 9721e1fb052680..be0b7ad1b73d36 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -44,17 +44,11 @@ static PyMemberDef union_members[] = { {0} }; - -// TODO; MM: Redo the implementation of this method; static PyObject * union_getattro(PyObject *self, PyObject *name) { - // printf("union_getattro"); unionobject *alias = (unionobject *) self; - // PyObject_Print(name, stdout, 0); - // printf("tp_name: (%s)\n", Py_TYPE(name)->tp_name); - return alias->args; - // return PyObject_GenericGetAttr(alias, name); + return PyObject_GenericGetAttr(alias, name); } // TODO: MM: Implement this for isinstance checks @@ -102,8 +96,8 @@ union_richcompare(PyObject *a, PyObject *b, int op) unionobject *aa = (unionobject *)a; if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); - printf("\n"); - PyObject_Print(b_args, stdout, 0); + printf("\n: Aa->args"); + // PyObject_Print(b_args, stdout, 0); PyObject_Print(aa->args, stdout, 0); return PyObject_RichCompare(aa->args, b_args, Py_EQ); } From 7f0018e0604f1af2278be6bd389502376182438e Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 16 Jul 2020 17:05:47 -0700 Subject: [PATCH 10/60] Reduce code duplication when adding union types together. --- Include/unionobject.h | 1 + Lib/test/test_isinstance.py | 20 ++++++++++---------- Objects/typeobject.c | 35 +++++++++++++++-------------------- Objects/unionobject.c | 34 +++++++++++++++++++++++++++------- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/Include/unionobject.h b/Include/unionobject.h index d72c3e88e0ca67..4fd972e48e37a2 100644 --- a/Include/unionobject.h +++ b/Include/unionobject.h @@ -5,6 +5,7 @@ extern "C" { #endif PyAPI_FUNC(PyObject *) Py_Union(PyObject *); +PyAPI_FUNC(PyObject *) Py_Union_AddToTuple(PyObject *, PyObject *, int); PyAPI_DATA(PyTypeObject) Py_UnionType; #ifdef __cplusplus diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 464242904ab1eb..59332a15a442fc 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -209,14 +209,14 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Super)) self.assertEqual(False, isinstance(AbstractChild(), Child)) - def test_isinstance_with_or_union(self): - self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) - self.assertEqual(False, isinstance(None, str | int)) - self.assertEqual(True, isinstance(3, str | int)) - self.assertEqual(True, isinstance("", str | int)) - self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) - self.assertEqual(True, isinstance(2, typing.List | int)) - self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) + # def test_isinstance_with_or_union(self): + # self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) + # self.assertEqual(False, isinstance(None, str | int)) + # self.assertEqual(True, isinstance(3, str | int)) + # self.assertEqual(True, isinstance("", str | int)) + # self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) + # self.assertEqual(True, isinstance(2, typing.List | int)) + # self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) def test_subclass_normal(self): # normal classes @@ -227,8 +227,8 @@ def test_subclass_normal(self): self.assertEqual(True, issubclass(Child, Child)) self.assertEqual(True, issubclass(Child, Super)) self.assertEqual(False, issubclass(Child, AbstractSuper)) - self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) - self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) + # self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) + # self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) def test_subclass_abstract(self): # abstract classes diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 276f21c5b2f0a3..586b0a01653b90 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3741,6 +3741,8 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } + +// TODO: MM: Move the following three functions? static int is_typing_name(PyObject *obj, char *name) { @@ -3758,7 +3760,6 @@ is_typing_name(PyObject *obj, char *name) return res; } -// TODO: MM: Move these? static int is_typevar(PyObject *obj) { @@ -3777,36 +3778,30 @@ type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias if ((param == NULL) || ( - (param != Py_None) && + (param != Py_None) & ! is_genericalias(param) && ! is_typevar(param) && - (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) + (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) && + (PyObject_IsInstance(param, (PyObject *) &Py_UnionType) != 1) ) ) { - PyObject_Print(param, stdout, 0); - PyObject_Print(self, stdout, 0); - printf("%d", PyObject_IsInstance(param, (PyObject *) &PyType_Type)); PyErr_SetString(PyExc_TypeError, "'type' expected"); return NULL; } - PyObject *tuple; - // 1. Create a tuple with types - if (PyObject_IsInstance((PyObject *)self, (PyObject *) &Py_UnionType) && PyObject_IsInstance((PyObject *)param, (PyObject *) &_PyNone_Type) != 1) { - PyObject* existingArgs = PyObject_GetAttrString((PyObject *)self, "__args__"); - int tuple_size = PyTuple_GET_SIZE(existingArgs); - tuple = PyTuple_New(tuple_size + 1); - for (Py_ssize_t iarg = 0; iarg < tuple_size; iarg++) { - PyObject* arg = PyTuple_GET_ITEM(existingArgs, iarg); - PyTuple_SET_ITEM(tuple, iarg, arg); - } - PyTuple_SET_ITEM(tuple, tuple_size, param); - Py_DECREF(existingArgs); + PyObject *tuple; + if (PyObject_IsInstance((PyObject *)self, (PyObject *) &Py_UnionType)) { + tuple = Py_Union_AddToTuple((PyObject *)self, param, 0); + } else if (PyObject_IsInstance(param, (PyObject *) &Py_UnionType)) { + tuple = Py_Union_AddToTuple(param, (PyObject *)self, 1); } else { - PyTypeObject *param_type = param == Py_None ? Py_TYPE(param) : param; - PyTypeObject *self_type = self == Py_None ? Py_TYPE(self) : self; + PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; + PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; tuple=PyTuple_Pack(2, self_type, param_type); + Py_DECREF(param_type); + Py_DECREF(self_type); } + PyObject *newUnionType=Py_Union(tuple); Py_DECREF(tuple); return newUnionType; diff --git a/Objects/unionobject.c b/Objects/unionobject.c index be0b7ad1b73d36..8ceed559b3b3a1 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -14,7 +14,6 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); - // Py_XDECREF(alias->origin); Py_XDECREF(alias->args); self->ob_type->tp_free(self); } @@ -47,8 +46,7 @@ static PyMemberDef union_members[] = { static PyObject * union_getattro(PyObject *self, PyObject *name) { - unionobject *alias = (unionobject *) self; - return PyObject_GenericGetAttr(alias, name); + return PyObject_GenericGetAttr(self, name); } // TODO: MM: Implement this for isinstance checks @@ -61,13 +59,13 @@ union_instancecheck(PyObject *self, PyObject *instance) for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); if (PyType_Check(arg)) { + // MM: TODO: Find out best method to use for this. retval = PyObject_IsInstance(instance, arg); } } return self; } - static int is_typing_name(PyObject *obj, char *name) { @@ -96,9 +94,6 @@ union_richcompare(PyObject *a, PyObject *b, int op) unionobject *aa = (unionobject *)a; if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); - printf("\n: Aa->args"); - // PyObject_Print(b_args, stdout, 0); - PyObject_Print(aa->args, stdout, 0); return PyObject_RichCompare(aa->args, b_args, Py_EQ); } PyTypeObject *type = Py_TYPE(b); @@ -152,3 +147,28 @@ Py_Union(PyObject *args) _PyObject_GC_TRACK(alias); return (PyObject *) alias; } + +// TODO: MM - If we can somehow sort the arguments reliably, we wouldn't need the boolean parameter here. +PyObject * +Py_Union_AddToTuple(PyObject *existing_param, PyObject *new_param, int addFirst) +{ + int position = 0; + PyObject* existingArgs = PyObject_GetAttrString((PyObject *)existing_param, "__args__"); + int tuple_size = PyTuple_GET_SIZE(existingArgs); + PyObject *tuple = PyTuple_New(tuple_size + 1); + + if (addFirst != 0) { + position = 1; + PyTuple_SET_ITEM(tuple, 0, (PyObject *)new_param); + } + + for (Py_ssize_t iarg = 0; iarg < tuple_size; iarg++) { + PyObject* arg = PyTuple_GET_ITEM(existingArgs, iarg); + PyTuple_SET_ITEM(tuple, iarg + position, arg); + } + + if (!addFirst) { + PyTuple_SET_ITEM(tuple, tuple_size, new_param); + } + return tuple; +} From cd584d59b2b5a2c836440b65391373d3e0d825b2 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 16 Jul 2020 20:38:24 -0700 Subject: [PATCH 11/60] Add __or__ method to _SpecialGenericAlias --- Lib/typing.py | 6 ++++++ Objects/abstract.c | 2 +- Objects/typeobject.c | 14 ++++---------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index d6f0e75f68b685..31912941637e7b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -798,6 +798,12 @@ def __subclasscheck__(self, cls): def __reduce__(self): return self._name + def __or__(self,right): + return Union[self,right] + def __ror__(self,right): + return Union[self,right] + def __invert__(self): + return Union[self,None] class _CallableGenericAlias(_GenericAlias, _root=True): def __repr__(self): diff --git a/Objects/abstract.c b/Objects/abstract.c index 745750c4b272f3..d6a1fcc470c1c8 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -910,7 +910,7 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) { PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.100s: " + "unsupported operand type(s) for () %.100s: " "'%.100s' and '%.100s'. Did you mean \"print(, " "file=)\"?", op_name, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 586b0a01653b90..a11096b2eed15e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3776,15 +3776,9 @@ is_genericalias(PyObject *obj) static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - if ((param == NULL) || - ( - (param != Py_None) & - ! is_genericalias(param) && - ! is_typevar(param) && - (PyObject_IsInstance(param, (PyObject *) &PyType_Type) != 1) && - (PyObject_IsInstance(param, (PyObject *) &Py_UnionType) != 1) - ) - ) { + if (param == NULL) + { + PyObject_Print(param, stdout, 0); PyErr_SetString(PyExc_TypeError, "'type' expected"); return NULL; } @@ -3801,7 +3795,7 @@ type_or(PyTypeObject* self, PyObject* param) { Py_DECREF(param_type); Py_DECREF(self_type); } - + PyObject_Print(tuple, stdout, 0); PyObject *newUnionType=Py_Union(tuple); Py_DECREF(tuple); return newUnionType; From ba031c15b66affff0c0b8a4b818db3f3908203bb Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 17 Jul 2020 11:41:18 -0700 Subject: [PATCH 12/60] Add support for NewType with unions. --- Objects/typeobject.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a11096b2eed15e..0efb51f5e1cad4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3772,13 +3772,35 @@ is_genericalias(PyObject *obj) return is_typing_name(obj, "GenericAlias"); } +static int +is_new_type(PyObject *obj) +{ + if (!PyObject_IsInstance((PyObject *)obj, (PyObject *)&PyFunction_Type)) { + return 0; + } + PyObject *module = PyObject_GetAttrString((PyObject *)obj, "__module__"); + if (module == NULL) { + return 0; + } + return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); +} + +static int +is_not_unionable (PyObject *obj) +{ + return (obj != Py_None) & + !is_genericalias(obj) && + !is_typevar(obj) && + !is_new_type(obj) && + (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) != 1) && + (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) != 1); +} static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - if (param == NULL) + if ((param == NULL) || is_not_unionable(param) || is_not_unionable(self)) { - PyObject_Print(param, stdout, 0); PyErr_SetString(PyExc_TypeError, "'type' expected"); return NULL; } @@ -3795,7 +3817,6 @@ type_or(PyTypeObject* self, PyObject* param) { Py_DECREF(param_type); Py_DECREF(self_type); } - PyObject_Print(tuple, stdout, 0); PyObject *newUnionType=Py_Union(tuple); Py_DECREF(tuple); return newUnionType; From e62de731cfae6c437e4da338647f46f7348829ae Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 17 Jul 2020 11:52:08 -0700 Subject: [PATCH 13/60] Remove old code from abstract.c --- Objects/abstract.c | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index d6a1fcc470c1c8..0e297e3cbcecfa 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2469,44 +2469,6 @@ abstract_issubclass(PyObject *derived, PyObject *cls) } } -static PyObject* -union_to_tuple(PyObject* cls) { - // if (!strcmp(Py_TYPE(cls)->tp_name, "typing.Union")) { - // PyObject* args = PyObject_GetAttrString(cls, "__args__"); - // PyObject_Print(args, stdout, 0); - // // return args; - // } - if (!strcmp(Py_TYPE(cls)->tp_name, "_GenericAlias")) { - PyObject* origin = PyObject_GetAttrString(cls, "__origin__"); - //printf("origin = %p\n",origin); - if (origin == NULL) { - printf("origin == NULL, return cls"); - return cls; - } - //printf("X origin = %s\n",Py_TYPE(cls)->tp_name); - if (PyObject_HasAttrString(origin, "_name")) { - PyObject* name = PyObject_GetAttrString(origin, "_name"); - if (name==NULL) { - printf("_name = NULL\n"); - Py_DECREF(origin); - return cls; - } - //printf("name = %s\n",Py_TYPE(name)->tp_name); - const char* data = (char*)PyUnicode_1BYTE_DATA(name); - //printf("DATA=%s\n",data); - if (data != NULL && !strcmp(data,"Union")) { - PyObject* new_cls = PyObject_GetAttrString(cls, "__args__"); - if (new_cls != NULL) { - printf("A down here."); - cls = new_cls; - } - } - Py_DECREF(name); - } - Py_DECREF(origin); - } - return cls; -} static int check_class(PyObject *cls, const char *error) @@ -2577,8 +2539,6 @@ object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls return object_isinstance(inst, cls); } - cls = union_to_tuple(cls); - if (PyTuple_Check(cls)) { /* Not a general sequence -- that opens up the road to recursion and stack overflow. */ @@ -2667,8 +2627,6 @@ object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls) return recursive_issubclass(derived, cls); } - cls = union_to_tuple(cls); - if (PyTuple_Check(cls)) { if (_Py_EnterRecursiveCall(tstate, " in __subclasscheck__")) { From f6ab05b8f04ea8ba4f86d5bf4b9a5cef58e06d6b Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 17 Jul 2020 13:27:37 -0700 Subject: [PATCH 14/60] Fix warning for incompatible pointer type. --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0efb51f5e1cad4..aaca8ead69d109 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3799,7 +3799,7 @@ is_not_unionable (PyObject *obj) static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - if ((param == NULL) || is_not_unionable(param) || is_not_unionable(self)) + if ((param == NULL) || is_not_unionable(param) || is_not_unionable((PyObject *)self)) { PyErr_SetString(PyExc_TypeError, "'type' expected"); return NULL; From 9c20fbc6e9949ee2ea01a231f717f7a58389add6 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Sun, 19 Jul 2020 11:05:15 -0700 Subject: [PATCH 15/60] Add support for union use in subclasses. --- Lib/test/test_isinstance.py | 26 +++++++++++++------------- Lib/typing.py | 9 +++++++++ Objects/abstract.c | 10 +++++++--- Objects/unionobject.c | 33 ++++++++++++++++++++++++++------- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 59332a15a442fc..cb34717544fba5 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -209,14 +209,14 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Super)) self.assertEqual(False, isinstance(AbstractChild(), Child)) - # def test_isinstance_with_or_union(self): - # self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) - # self.assertEqual(False, isinstance(None, str | int)) - # self.assertEqual(True, isinstance(3, str | int)) - # self.assertEqual(True, isinstance("", str | int)) - # self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) - # self.assertEqual(True, isinstance(2, typing.List | int)) - # self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) + def test_isinstance_with_or_union(self): + # self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) + self.assertEqual(False, isinstance(None, str | int)) + self.assertEqual(True, isinstance(3, str | int)) + self.assertEqual(True, isinstance("", str | int)) + self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) + self.assertEqual(True, isinstance(2, typing.List | int)) + self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) def test_subclass_normal(self): # normal classes @@ -227,8 +227,8 @@ def test_subclass_normal(self): self.assertEqual(True, issubclass(Child, Child)) self.assertEqual(True, issubclass(Child, Super)) self.assertEqual(False, issubclass(Child, AbstractSuper)) - # self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) - # self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) + self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) + self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) def test_subclass_abstract(self): # abstract classes @@ -263,9 +263,9 @@ def test_isinstance_recursion_limit(self): # blown self.assertRaises(RecursionError, blowstack, isinstance, '', str) - # def test_subclass_with_union(self): - # self.assertEqual(True, issubclass(int, int | float | int)) - # self.assertEqual(True, issubclass(str, str | Child | str)) + def test_subclass_with_union(self): + self.assertEqual(True, issubclass(int, int | float | int)) + self.assertEqual(True, issubclass(str, str | Child | str)) def test_issubclass_refcount_handling(self): # bpo-39382: abstract_issubclass() didn't hold item reference while diff --git a/Lib/typing.py b/Lib/typing.py index 31912941637e7b..af91c6528f53ad 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -890,6 +890,15 @@ def __repr__(self): return f'typing.Optional[{_type_repr(args[0])}]' return super().__repr__() + def __instancecheck__(self, obj): + return self.__subclasscheck__(type(obj)) + + def __subclasscheck__(self, cls): + for arg in self.__args__: + if issubclass(cls, arg): + return True + + class Generic: """Abstract base class for generic types. diff --git a/Objects/abstract.c b/Objects/abstract.c index 0e297e3cbcecfa..cd8ea9c6e85506 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2605,10 +2605,14 @@ recursive_issubclass(PyObject *derived, PyObject *cls) if (!check_class(derived, "issubclass() arg 1 must be a class")) return -1; - if (!check_class(cls, - "issubclass() arg 2 must be a class" - " or tuple of classes")) + + PyTypeObject *type = Py_TYPE(cls); + int is_union = (PyType_Check(type) && strcmp(type->tp_name, "typing.Union") == 0); + if (!is_union && !check_class(cls, + "issubclass() arg 2 must be a class" + " or tuple of classes")) { return -1; + } return abstract_issubclass(derived, cls); } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 8ceed559b3b3a1..5789acd3e674a2 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -49,21 +49,38 @@ union_getattro(PyObject *self, PyObject *name) return PyObject_GenericGetAttr(self, name); } -// TODO: MM: Implement this for isinstance checks static PyObject * union_instancecheck(PyObject *self, PyObject *instance) { unionobject *alias = (unionobject *) self; Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); - int retval; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); if (PyType_Check(arg)) { - // MM: TODO: Find out best method to use for this. - retval = PyObject_IsInstance(instance, arg); + if (PyObject_IsInstance(instance, arg) != 0) + { + return self; + } } } - return self; + return instance; +} + +static PyObject * +union_subclasscheck(PyObject *self, PyObject *instance) +{ + unionobject *alias = (unionobject *) self; + Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (PyType_Check(arg)) { + if (PyType_IsSubtype(instance, arg) != 0) + { + return self; + } + } + } + return instance; } static int @@ -85,8 +102,8 @@ is_typing_name(PyObject *obj, char *name) static PyMethodDef union_methods[] = { {"__instancecheck__", union_instancecheck, METH_O}, - {0} -}; + {"__subclasscheck__", union_subclasscheck, METH_O}, + {0}}; static PyObject * union_richcompare(PyObject *a, PyObject *b, int op) @@ -105,6 +122,8 @@ union_richcompare(PyObject *a, PyObject *b, int op) } + + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "typing.Union", From 9f4898972947e3176924dbbc178ca669c383c6dd Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 22 Jul 2020 14:45:42 -0700 Subject: [PATCH 16/60] Fix code style inconsistencies. --- Lib/test/test_isinstance.py | 20 ++++++++++---------- Lib/test/test_types.py | 16 ++++++++-------- Lib/typing.py | 31 ++++++++++++++----------------- Objects/abstract.c | 4 ++-- Objects/unionobject.c | 3 --- 5 files changed, 34 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index cb34717544fba5..5fb0f7da172c76 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -211,12 +211,12 @@ def test_isinstance_abstract(self): def test_isinstance_with_or_union(self): # self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) - self.assertEqual(False, isinstance(None, str | int)) - self.assertEqual(True, isinstance(3, str | int)) - self.assertEqual(True, isinstance("", str | int)) - self.assertEqual(True, isinstance([], typing.List | typing.Tuple)) - self.assertEqual(True, isinstance(2, typing.List | int)) - self.assertEqual(False, isinstance(2, typing.List | typing.Tuple)) + self.assertFalse(isinstance(None, str | int)) + self.assertTrue(isinstance(3, str | int)) + self.assertTrue(isinstance("", str | int)) + self.assertTrue(isinstance([], typing.List | typing.Tuple)) + self.assertTrue(isinstance(2, typing.List | int)) + self.assertFalse(isinstance(2, typing.List | typing.Tuple)) def test_subclass_normal(self): # normal classes @@ -227,8 +227,8 @@ def test_subclass_normal(self): self.assertEqual(True, issubclass(Child, Child)) self.assertEqual(True, issubclass(Child, Super)) self.assertEqual(False, issubclass(Child, AbstractSuper)) - self.assertEqual(True, issubclass(typing.List, typing.List|typing.Tuple)) - self.assertEqual(False, issubclass(int, typing.List|typing.Tuple)) + self.assertTrue(issubclass(typing.List, typing.List|typing.Tuple)) + self.assertFalse(issubclass(int, typing.List|typing.Tuple)) def test_subclass_abstract(self): # abstract classes @@ -264,8 +264,8 @@ def test_isinstance_recursion_limit(self): self.assertRaises(RecursionError, blowstack, isinstance, '', str) def test_subclass_with_union(self): - self.assertEqual(True, issubclass(int, int | float | int)) - self.assertEqual(True, issubclass(str, str | Child | str)) + self.assertTrue(issubclass(int, int | float | int)) + self.assertTrue(issubclass(str, str | Child | str)) def test_issubclass_refcount_handling(self): # bpo-39382: abstract_issubclass() didn't hold item reference while diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 59bc40dd64d419..46c423b4eeab8e 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -611,14 +611,14 @@ def test_or_types_operator(self): self.assertEqual(typing.List[int] | None, typing.Union[typing.List[int], None]) self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) self.assertEqual( - BaseException | \ - bool | \ - bytes | \ - complex | \ - float | \ - int | \ - list | \ - map | \ + BaseException | + bool | + bytes | + complex | + float | + int | + list | + map | set, typing.Union[ BaseException, diff --git a/Lib/typing.py b/Lib/typing.py index af91c6528f53ad..d7b8afdc119a7f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -578,12 +578,11 @@ def __init__(self, name, *constraints, bound=None, if def_mod != 'typing': self.__module__ = def_mod - def __or__(self,right): - return Union[self,right] - def __ror__(self,right): - return Union[self,right] - def __invert__(self): - return Union[self,None] + def __or__(self, right): + return Union[self, right] + + def __ror__(self, right): + return Union[self, right] def __repr__(self): if self.__covariant__: @@ -692,12 +691,11 @@ def __eq__(self, other): def __hash__(self): return hash((self.__origin__, self.__args__)) - def __or__(self,right): - return Union[self,right] - def __ror__(self,right): - return Union[self,right] - def __invert__(self): - return Union[self,None] + def __or__(self, right): + return Union[self, right] + + def __ror__(self, right): + return Union[self, right] @_tp_cache def __getitem__(self, params): @@ -798,12 +796,11 @@ def __subclasscheck__(self, cls): def __reduce__(self): return self._name - def __or__(self,right): + def __or__(self, right): return Union[self,right] - def __ror__(self,right): - return Union[self,right] - def __invert__(self): - return Union[self,None] + + def __ror__(self, right): + return Union[self, right] class _CallableGenericAlias(_GenericAlias, _root=True): def __repr__(self): diff --git a/Objects/abstract.c b/Objects/abstract.c index cd8ea9c6e85506..24559263f44b3f 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -910,7 +910,7 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) { PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for () %.100s: " + "unsupported operand type(s) for %.100s: " "'%.100s' and '%.100s'. Did you mean \"print(, " "file=)\"?", op_name, @@ -2512,7 +2512,7 @@ object_isinstance(PyObject *inst, PyObject *cls) } else { if (!check_class(cls, - "isinstance() arg 2 must be a type, a tuple of types or union")) + "isinstance() arg 2 must be a type, a tuple of types or a union")) return -1; retval = _PyObject_LookupAttrId(inst, &PyId___class__, &icls); if (icls != NULL) { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 5789acd3e674a2..38457ea36f7542 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -121,9 +121,6 @@ union_richcompare(PyObject *a, PyObject *b, int op) Py_RETURN_FALSE; } - - - PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "typing.Union", From a0cd651f70d0c0476afe488d98d90906ac3a7ed2 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 22 Jul 2020 16:42:43 -0700 Subject: [PATCH 17/60] Change is_not_union -> is_union --- Objects/typeobject.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index aaca8ead69d109..f97f2b7150ce47 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3775,10 +3775,10 @@ is_genericalias(PyObject *obj) static int is_new_type(PyObject *obj) { - if (!PyObject_IsInstance((PyObject *)obj, (PyObject *)&PyFunction_Type)) { + if (!PyObject_IsInstance(obj, (PyObject *)&PyFunction_Type)) { return 0; } - PyObject *module = PyObject_GetAttrString((PyObject *)obj, "__module__"); + PyObject *module = PyObject_GetAttrString(obj, "__module__"); if (module == NULL) { return 0; } @@ -3786,23 +3786,27 @@ is_new_type(PyObject *obj) } static int -is_not_unionable (PyObject *obj) +is_unionable(PyObject *obj) { - return (obj != Py_None) & - !is_genericalias(obj) && - !is_typevar(obj) && - !is_new_type(obj) && - (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) != 1) && - (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) != 1); + if (obj == Py_None) { + return 1; + } + return ( + is_genericalias(obj) || + is_typevar(obj) || + is_new_type(obj) || + (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || + (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); } static PyObject * type_or(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - if ((param == NULL) || is_not_unionable(param) || is_not_unionable((PyObject *)self)) + if ((param == NULL) || !is_unionable(param) || !is_unionable((PyObject *)self)) { - PyErr_SetString(PyExc_TypeError, "'type' expected"); - return NULL; + Py_INCREF(Py_NotImplemented); + // PyErr_SetString(Py_NotImplemented, "'type' expected"); + return Py_NotImplemented; } PyObject *tuple; From a5c6850c7a496a081c99d2732fe6373dceefe7f8 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 24 Jul 2020 12:05:54 -0700 Subject: [PATCH 18/60] Implement dedup and flatten. --- Include/unionobject.h | 2 +- Lib/test/test_types.py | 1 + Lib/typing.py | 3 +- Objects/abstract.c | 5 +- Objects/typeobject.c | 83 +---------------- Objects/unionobject.c | 200 ++++++++++++++++++++++++++++++++++------- 6 files changed, 178 insertions(+), 116 deletions(-) diff --git a/Include/unionobject.h b/Include/unionobject.h index 4fd972e48e37a2..27642373d04f49 100644 --- a/Include/unionobject.h +++ b/Include/unionobject.h @@ -5,7 +5,7 @@ extern "C" { #endif PyAPI_FUNC(PyObject *) Py_Union(PyObject *); -PyAPI_FUNC(PyObject *) Py_Union_AddToTuple(PyObject *, PyObject *, int); +PyAPI_FUNC(PyObject *) Py_Union_New(PyTypeObject *, PyObject *); PyAPI_DATA(PyTypeObject) Py_UnionType; #ifdef __cplusplus diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 46c423b4eeab8e..bd3b8ada1a1d6c 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -602,6 +602,7 @@ def test_method_descriptor_types(self): def test_or_types_operator(self): self.assertEqual(int | str, typing.Union[int, str]) + self.assertEqual(str | int, typing.Union[int, str]) self.assertEqual(int | None, typing.Union[int, None]) # self.assertEqual(None | int, typing.Union[int, None]) self.assertEqual(int | str | list, typing.Union[int, str, list]) diff --git a/Lib/typing.py b/Lib/typing.py index d7b8afdc119a7f..11a4d555c4a999 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -117,6 +117,7 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. +Union = type(int|str) def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). @@ -797,7 +798,7 @@ def __reduce__(self): return self._name def __or__(self, right): - return Union[self,right] + return Union[self, right] def __ror__(self, right): return Union[self, right] diff --git a/Objects/abstract.c b/Objects/abstract.c index 24559263f44b3f..cff31652e4770a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -864,9 +864,11 @@ binary_op1(PyObject *v, PyObject *w, const int op_slot) if (slotw == slotv) slotw = NULL; } + if (slotv) { if (slotw && PyType_IsSubtype(Py_TYPE(w), Py_TYPE(v))) { x = slotw(v, w); + PyObject_Print(x, stdout, 0); if (x != Py_NotImplemented) return x; Py_DECREF(x); /* can't do it */ @@ -918,7 +920,6 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) Py_TYPE(w)->tp_name); return NULL; } - return binop_type_error(v, w, op_name); } return result; @@ -2607,7 +2608,7 @@ recursive_issubclass(PyObject *derived, PyObject *cls) return -1; PyTypeObject *type = Py_TYPE(cls); - int is_union = (PyType_Check(type) && strcmp(type->tp_name, "typing.Union") == 0); + int is_union = (PyType_Check(type) && strcmp(type->tp_name, "types.Union") == 0); if (!is_union && !check_class(cls, "issubclass() arg 2 must be a class" " or tuple of classes")) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f97f2b7150ce47..4bed7b8bad6224 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3741,89 +3741,10 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } - -// TODO: MM: Move the following three functions? -static int -is_typing_name(PyObject *obj, char *name) -{ - PyTypeObject *type = Py_TYPE(obj); - if (strcmp(type->tp_name, name) != 0) { - return 0; - } - PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); - if (module == NULL) { - return -1; - } - int res = PyUnicode_Check(module) - && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - return res; -} - -static int -is_typevar(PyObject *obj) -{ - return is_typing_name(obj, "TypeVar"); -} - -static int -is_genericalias(PyObject *obj) -{ - return is_typing_name(obj, "GenericAlias"); -} - -static int -is_new_type(PyObject *obj) -{ - if (!PyObject_IsInstance(obj, (PyObject *)&PyFunction_Type)) { - return 0; - } - PyObject *module = PyObject_GetAttrString(obj, "__module__"); - if (module == NULL) { - return 0; - } - return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); -} - -static int -is_unionable(PyObject *obj) -{ - if (obj == Py_None) { - return 1; - } - return ( - is_genericalias(obj) || - is_typevar(obj) || - is_new_type(obj) || - (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || - (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); -} - static PyObject * type_or(PyTypeObject* self, PyObject* param) { - // Check param is a PyType or GenericAlias - if ((param == NULL) || !is_unionable(param) || !is_unionable((PyObject *)self)) - { - Py_INCREF(Py_NotImplemented); - // PyErr_SetString(Py_NotImplemented, "'type' expected"); - return Py_NotImplemented; - } - - PyObject *tuple; - if (PyObject_IsInstance((PyObject *)self, (PyObject *) &Py_UnionType)) { - tuple = Py_Union_AddToTuple((PyObject *)self, param, 0); - } else if (PyObject_IsInstance(param, (PyObject *) &Py_UnionType)) { - tuple = Py_Union_AddToTuple(param, (PyObject *)self, 1); - } else { - PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; - PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; - tuple=PyTuple_Pack(2, self_type, param_type); - Py_DECREF(param_type); - Py_DECREF(self_type); - } - PyObject *newUnionType=Py_Union(tuple); - Py_DECREF(tuple); - return newUnionType; + PyObject* new_union = Py_Union_New(self, param); + return new_union; } static PyNumberMethods type_as_number = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 38457ea36f7542..ff1045fb0f319a 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -3,6 +3,7 @@ #include "pycore_object.h" #include "structmember.h" + typedef struct { PyObject_HEAD; PyObject *args; @@ -18,7 +19,6 @@ unionobject_dealloc(PyObject *self) self->ob_type->tp_free(self); } - static Py_hash_t union_hash(PyObject *self) { @@ -74,7 +74,7 @@ union_subclasscheck(PyObject *self, PyObject *instance) for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); if (PyType_Check(arg)) { - if (PyType_IsSubtype(instance, arg) != 0) + if (PyType_IsSubtype((PyTypeObject *)instance, (PyTypeObject *)arg) != 0) { return self; } @@ -108,22 +108,176 @@ static PyMethodDef union_methods[] = { static PyObject * union_richcompare(PyObject *a, PyObject *b, int op) { + PyObject* b_set = NULL; unionobject *aa = (unionobject *)a; + PyObject* a_set = PySet_New(aa->args); + if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); - return PyObject_RichCompare(aa->args, b_args, Py_EQ); + b_set = PySet_New(b_args); + } + PyTypeObject *type = Py_TYPE(b); - if (strcmp(type->tp_name, "typing.Union") != 0) { - unionobject *bb = (unionobject *)a; - return PyObject_RichCompare(aa->args, bb->args, Py_EQ); + if (type == &Py_UnionType) { + unionobject *bb = (unionobject *)b; + b_set = PySet_New(bb->args); + } + + if (b_set == NULL) { + Py_RETURN_FALSE; + } + + return PyObject_RichCompare(a_set, b_set, Py_EQ); +} + +static PyObject* +flatten_args(PyObject* args) { + int arg_length = PyTuple_GET_SIZE(args); + int total_args = 0; + // Get number of args once it's flattened. + for (int i = 0; i < arg_length; i++) { + PyObject *arg = PyTuple_GET_ITEM(args, i); + PyTypeObject* arg_type = Py_TYPE(arg); + if (arg_type == &Py_UnionType) { + unionobject *alias = (unionobject *)arg; + total_args = total_args + PyTuple_GET_SIZE(alias->args); + } else { + total_args++; + } } - Py_RETURN_FALSE; + // Create new tuple of flattened args. + PyObject *flattened_args = PyTuple_New(total_args); + int pos = 0; + for (int i = 0; i < arg_length; i++) { + PyObject *arg = PyTuple_GET_ITEM(args, i); + PyTypeObject* arg_type = Py_TYPE(arg); + if (arg_type == &Py_UnionType) { + unionobject *alias = (unionobject *)arg; + PyObject* nested_args = alias->args; + int nested_arg_length = PyTuple_GET_SIZE(nested_args); + for (int j = 0; j < nested_arg_length; j++) { + PyObject* nested_arg = PyTuple_GET_ITEM(nested_args, j); + PyTuple_SET_ITEM(flattened_args, pos, nested_arg); + pos++; + } + } else { + PyTuple_SET_ITEM(flattened_args, pos, arg); + pos++; + } + } + + return flattened_args; +} + +static PyObject* +dedup_and_flatten_args(PyObject* args) { + args = flatten_args(args); + int arg_length = PyTuple_GET_SIZE(args); + PyObject* temp[arg_length]; + for (int i = 0; i < arg_length ; i++) { temp[i] = NULL; } + + // Add unique elements to an array. + int temp_count = 0; + for(int i = 0; i < arg_length; i++) { + int is_duplicate = 0; + PyObject* i_tuple = PyTuple_GET_ITEM(args, i); + for(int j = i + 1; j < arg_length; j++) { + PyObject* j_tuple = PyTuple_GET_ITEM(args, j); + if (i_tuple == j_tuple) is_duplicate = 1; + } + if (!is_duplicate) { + temp[temp_count] = i_tuple; + temp_count++; + } + } + + // Create new tuple from deduplicated array. + PyObject* temp_arg = NULL; + PyObject* new_args = PyTuple_New(temp_count); + for(int k = 0; k < arg_length; k++) { + temp_arg = temp[k]; + + if (temp_arg != NULL) + PyTuple_SET_ITEM(new_args, k, temp_arg); + } + + return new_args; +} + + +static int +is_typevar(PyObject *obj) +{ + return is_typing_name(obj, "TypeVar"); +} + +static int +is_genericalias(PyObject *obj) +{ + return is_typing_name(obj, "GenericAlias"); } +static int +is_new_type(PyObject *obj) +{ + if (!PyObject_IsInstance(obj, (PyObject *)&PyFunction_Type)) { + return 0; + } + PyObject *module = PyObject_GetAttrString(obj, "__module__"); + if (module == NULL) { + return 0; + } + return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); +} + +static int +is_unionable(PyObject *obj) +{ + if (obj == Py_None) { + return 1; + } + return ( + is_genericalias(obj) || + is_typevar(obj) || + is_new_type(obj) || + (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || + (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); +} + +static PyObject * +union_new(PyTypeObject* self, PyObject* param) { + // Check param is a PyType or GenericAlias + if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject *)self)) + { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; + PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; + PyObject *tuple=PyTuple_Pack(2, self_type, param_type); + PyObject *newUnionType = Py_Union(tuple); + Py_DECREF(param_type); + Py_DECREF(self_type); + Py_DECREF(tuple); + return newUnionType; +} + + +static PyObject * +type_or(PyTypeObject* self, PyObject* param) { + return union_new(self, param); +} + +static PyNumberMethods union_as_number = { + .nb_or = (binaryfunc)type_or, // Add __or__ function +}; + + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "typing.Union", + .tp_name = "types.Union", .tp_doc = "Represent a PEP 604 union type\n" "\n" "E.g. for int | str", @@ -138,8 +292,10 @@ PyTypeObject Py_UnionType = { .tp_members = union_members, .tp_methods = union_methods, .tp_richcompare = union_richcompare, + .tp_as_number = &union_as_number, }; + PyObject * Py_Union(PyObject *args) { @@ -159,32 +315,14 @@ Py_Union(PyObject *args) return NULL; } - alias->args = args; + PyObject* new_args = dedup_and_flatten_args(args); + alias->args = new_args; _PyObject_GC_TRACK(alias); return (PyObject *) alias; } -// TODO: MM - If we can somehow sort the arguments reliably, we wouldn't need the boolean parameter here. PyObject * -Py_Union_AddToTuple(PyObject *existing_param, PyObject *new_param, int addFirst) -{ - int position = 0; - PyObject* existingArgs = PyObject_GetAttrString((PyObject *)existing_param, "__args__"); - int tuple_size = PyTuple_GET_SIZE(existingArgs); - PyObject *tuple = PyTuple_New(tuple_size + 1); - - if (addFirst != 0) { - position = 1; - PyTuple_SET_ITEM(tuple, 0, (PyObject *)new_param); - } - - for (Py_ssize_t iarg = 0; iarg < tuple_size; iarg++) { - PyObject* arg = PyTuple_GET_ITEM(existingArgs, iarg); - PyTuple_SET_ITEM(tuple, iarg + position, arg); - } - - if (!addFirst) { - PyTuple_SET_ITEM(tuple, tuple_size, new_param); - } - return tuple; +Py_Union_New(PyTypeObject* self, PyObject* param) { + PyObject* new_union = union_new(self, param); + return new_union; } From 5255f4f9688cfcff3ad851e32d3842e63c3db5b9 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 24 Jul 2020 13:55:47 -0700 Subject: [PATCH 19/60] Add basic tp_repr to unionobject. --- Lib/test/test_isinstance.py | 3 +- Lib/test/test_types.py | 4 +- Objects/unionobject.c | 106 ++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 5fb0f7da172c76..5681372796e161 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -210,13 +210,14 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Child)) def test_isinstance_with_or_union(self): - # self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) + self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) self.assertFalse(isinstance(None, str | int)) self.assertTrue(isinstance(3, str | int)) self.assertTrue(isinstance("", str | int)) self.assertTrue(isinstance([], typing.List | typing.Tuple)) self.assertTrue(isinstance(2, typing.List | int)) self.assertFalse(isinstance(2, typing.List | typing.Tuple)) + self.assertTrue(isinstance(None, int |None)) def test_subclass_normal(self): # normal classes diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index bd3b8ada1a1d6c..3b099e3ef2be2a 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -604,13 +604,15 @@ def test_or_types_operator(self): self.assertEqual(int | str, typing.Union[int, str]) self.assertEqual(str | int, typing.Union[int, str]) self.assertEqual(int | None, typing.Union[int, None]) - # self.assertEqual(None | int, typing.Union[int, None]) + self.assertEqual(None | int, typing.Union[int, None]) self.assertEqual(int | str | list, typing.Union[int, str, list]) self.assertEqual(int | (str | list), typing.Union[int, str, list]) + self.assertEqual(str | (int | list), typing.Union[int, str, list]) self.assertEqual(typing.List | typing.Tuple, typing.Union[typing.List, typing.Tuple]) self.assertEqual(typing.List[int] | typing.Tuple[int], typing.Union[typing.List[int], typing.Tuple[int]]) self.assertEqual(typing.List[int] | None, typing.Union[typing.List[int], None]) self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) + self.assertEqual(str | float | int | complex | int == (int | str) | (float | complex)) self.assertEqual( BaseException | bool | diff --git a/Objects/unionobject.c b/Objects/unionobject.c index ff1045fb0f319a..2c2cf87e394af8 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -274,6 +274,110 @@ static PyNumberMethods union_as_number = { .nb_or = (binaryfunc)type_or, // Add __or__ function }; +static int +union_repr_item(_PyUnicodeWriter *writer, PyObject *p) +{ + _Py_IDENTIFIER(__module__); + _Py_IDENTIFIER(__qualname__); + _Py_IDENTIFIER(__origin__); + _Py_IDENTIFIER(__args__); + PyObject *qualname = NULL; + PyObject *module = NULL; + PyObject *r = NULL; + PyObject *tmp; + int err; + + if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + goto done; + } + + if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { + goto done; + } + if (tmp != NULL) { + Py_DECREF(tmp); + if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) { + goto done; + } + if (tmp != NULL) { + Py_DECREF(tmp); + // It looks like a GenericAlias + goto use_repr; + } + } + + if (_PyObject_LookupAttrId(p, &PyId___qualname__, &qualname) < 0) { + goto done; + } + if (qualname == NULL) { + goto use_repr; + } + if (_PyObject_LookupAttrId(p, &PyId___module__, &module) < 0) { + goto done; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + r = PyObject_Str(qualname); + goto done; + } + else { + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto done; + } + +use_repr: + r = PyObject_Repr(p); + +done: + Py_XDECREF(qualname); + Py_XDECREF(module); + if (r == NULL) { + // error if any of the above PyObject_Repr/PyUnicode_From* fail + err = -1; + } + else { + err = _PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); + } + return err; +} + +static PyObject * +union_repr(PyObject *self) +{ + unionobject *alias = (unionobject *)self; + Py_ssize_t len = PyTuple_GET_SIZE(alias->args); + + + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0) { + if (_PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { + goto error; + } + } + PyObject *p = PyTuple_GET_ITEM(alias->args, i); + if (union_repr_item(&writer, p) < 0) { + goto error; + } + } + return _PyUnicodeWriter_Finish(&writer); +error: + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + +} + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -293,9 +397,11 @@ PyTypeObject Py_UnionType = { .tp_methods = union_methods, .tp_richcompare = union_richcompare, .tp_as_number = &union_as_number, + .tp_repr = union_repr, }; + PyObject * Py_Union(PyObject *args) { From 1732fad2989e2e2232fb0fefd3c08a95ce0c7a67 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 24 Jul 2020 16:34:50 -0700 Subject: [PATCH 20/60] Update is_unionable. --- Lib/test/test_isinstance.py | 4 ++-- Lib/test/test_types.py | 2 +- Objects/unionobject.c | 18 ++++++++++++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 5681372796e161..6e2b18a406892b 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -210,14 +210,14 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Child)) def test_isinstance_with_or_union(self): - self.assertEqual(True, isinstance(AbstractChild(), AbstractChild | int)) + # self.assertTrue(isinstance(AbstractChild(), AbstractChild | int)) self.assertFalse(isinstance(None, str | int)) self.assertTrue(isinstance(3, str | int)) self.assertTrue(isinstance("", str | int)) self.assertTrue(isinstance([], typing.List | typing.Tuple)) self.assertTrue(isinstance(2, typing.List | int)) self.assertFalse(isinstance(2, typing.List | typing.Tuple)) - self.assertTrue(isinstance(None, int |None)) + self.assertTrue(isinstance(None, int | None)) def test_subclass_normal(self): # normal classes diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3b099e3ef2be2a..2a3eb60664bb08 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -612,7 +612,7 @@ def test_or_types_operator(self): self.assertEqual(typing.List[int] | typing.Tuple[int], typing.Union[typing.List[int], typing.Tuple[int]]) self.assertEqual(typing.List[int] | None, typing.Union[typing.List[int], None]) self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) - self.assertEqual(str | float | int | complex | int == (int | str) | (float | complex)) + self.assertEqual(str | float | int | complex | int, (int | str) | (float | complex)) self.assertEqual( BaseException | bool | diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 2c2cf87e394af8..c9f4261f3af717 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -231,6 +231,19 @@ is_new_type(PyObject *obj) return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); } +static int +is_class(PyObject *obj) +{ + if (!PyObject_HasAttrString(obj, "__module__")) { + return 0; + } + PyObject *module = PyObject_GetAttrString(obj, "__module__"); + if (module == NULL) { + return 0; + } + return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "__main__"); +} + static int is_unionable(PyObject *obj) { @@ -241,14 +254,15 @@ is_unionable(PyObject *obj) is_genericalias(obj) || is_typevar(obj) || is_new_type(obj) || + is_class(obj) || (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || - (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); + (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); } static PyObject * union_new(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias - if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject *)self)) + if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; From 31653dcc0c112cf0ca292e9b7ee3b4487afd3d67 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Sat, 25 Jul 2020 14:12:41 -0700 Subject: [PATCH 21/60] Clean up whitespace. --- Lib/test/test_typing.py | 6 ------ Objects/abstract.c | 8 ++------ Objects/unionobject.c | 16 ++-------------- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index e52ce095345fca..7589b831711ea0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -244,8 +244,6 @@ def test_subclass_error(self): issubclass(int, Union) with self.assertRaises(TypeError): issubclass(Union, int) - # with self.assertRaises(TypeError): - # issubclass(int, Union[int, str]) with self.assertRaises(TypeError): issubclass(Union[int, str], int) @@ -347,10 +345,6 @@ def test_empty(self): with self.assertRaises(TypeError): Union[()] - # def test_union_instance_type_error(self): - # with self.assertRaises(TypeError): - # isinstance(42, Union[int, str]) - def test_no_eval_union(self): u = Union[int, str] def f(x: u): ... diff --git a/Objects/abstract.c b/Objects/abstract.c index cff31652e4770a..c804948756ca5a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -864,11 +864,9 @@ binary_op1(PyObject *v, PyObject *w, const int op_slot) if (slotw == slotv) slotw = NULL; } - if (slotv) { if (slotw && PyType_IsSubtype(Py_TYPE(w), Py_TYPE(v))) { x = slotw(v, w); - PyObject_Print(x, stdout, 0); if (x != Py_NotImplemented) return x; Py_DECREF(x); /* can't do it */ @@ -2470,7 +2468,6 @@ abstract_issubclass(PyObject *derived, PyObject *cls) } } - static int check_class(PyObject *cls, const char *error) { @@ -2503,7 +2500,6 @@ object_isinstance(PyObject *inst, PyObject *cls) (PyTypeObject *)icls, (PyTypeObject *)cls); } - else { retval = 0; } @@ -2608,10 +2604,10 @@ recursive_issubclass(PyObject *derived, PyObject *cls) return -1; PyTypeObject *type = Py_TYPE(cls); - int is_union = (PyType_Check(type) && strcmp(type->tp_name, "types.Union") == 0); + int is_union = (PyType_Check(type) && type == &Py_UnionType); if (!is_union && !check_class(cls, "issubclass() arg 2 must be a class" - " or tuple of classes")) { + " a tuple of classes, or a union.")) { return -1; } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index c9f4261f3af717..06719f882613a1 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -231,19 +231,6 @@ is_new_type(PyObject *obj) return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); } -static int -is_class(PyObject *obj) -{ - if (!PyObject_HasAttrString(obj, "__module__")) { - return 0; - } - PyObject *module = PyObject_GetAttrString(obj, "__module__"); - if (module == NULL) { - return 0; - } - return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "__main__"); -} - static int is_unionable(PyObject *obj) { @@ -254,7 +241,7 @@ is_unionable(PyObject *obj) is_genericalias(obj) || is_typevar(obj) || is_new_type(obj) || - is_class(obj) || + PyType_Check(obj) || (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); } @@ -264,6 +251,7 @@ union_new(PyTypeObject* self, PyObject* param) { // Check param is a PyType or GenericAlias if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) { + Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } From 2ffb0508f11ff42f44c19133f886ff0f80fe80f4 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Sat, 25 Jul 2020 14:17:30 -0700 Subject: [PATCH 22/60] Update test_types. --- Lib/test/test_isinstance.py | 2 +- Lib/test/test_types.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 6e2b18a406892b..4a53b2de33d156 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -210,7 +210,7 @@ def test_isinstance_abstract(self): self.assertEqual(False, isinstance(AbstractChild(), Child)) def test_isinstance_with_or_union(self): - # self.assertTrue(isinstance(AbstractChild(), AbstractChild | int)) + self.assertTrue(isinstance(Super(), Super | int)) self.assertFalse(isinstance(None, str | int)) self.assertTrue(isinstance(3, str | int)) self.assertTrue(isinstance("", str | int)) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 2a3eb60664bb08..0fa91937f2282c 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -12,6 +12,9 @@ import weakref import typing +class Example: + pass + class TypesTests(unittest.TestCase): def test_truth_values(self): @@ -638,6 +641,9 @@ def test_or_types_operator(self): int | 3 with self.assertRaises(TypeError): 3 | int + with self.assertRaises(TypeError): + Example() | int + def test_or_type_operator_with_TypeVar(self): TV = typing.TypeVar('T') From 3fed944bc37a8e03bd0cfe121dfc5c1161c400ab Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 09:47:00 -0700 Subject: [PATCH 23/60] Fix style and grammar issues. --- Lib/typing.py | 1 - Objects/abstract.c | 2 +- Objects/typeobject.c | 3 +-- Objects/unionobject.c | 29 +++++++++++++---------------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 11a4d555c4a999..2d2d7e3172280d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -117,7 +117,6 @@ # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. -Union = type(int|str) def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). diff --git a/Objects/abstract.c b/Objects/abstract.c index c804948756ca5a..3744bf67517180 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2606,7 +2606,7 @@ recursive_issubclass(PyObject *derived, PyObject *cls) PyTypeObject *type = Py_TYPE(cls); int is_union = (PyType_Check(type) && type == &Py_UnionType); if (!is_union && !check_class(cls, - "issubclass() arg 2 must be a class" + "issubclass() arg 2 must be a class," " a tuple of classes, or a union.")) { return -1; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4bed7b8bad6224..4291abe4323b3a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3743,8 +3743,7 @@ type_is_gc(PyTypeObject *type) static PyObject * type_or(PyTypeObject* self, PyObject* param) { - PyObject* new_union = Py_Union_New(self, param); - return new_union; + return Py_Union_New(self, param); } static PyNumberMethods type_as_number = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 06719f882613a1..b103710a1afa57 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -115,7 +115,6 @@ union_richcompare(PyObject *a, PyObject *b, int op) if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); b_set = PySet_New(b_args); - } PyTypeObject *type = Py_TYPE(b); @@ -132,7 +131,8 @@ union_richcompare(PyObject *a, PyObject *b, int op) } static PyObject* -flatten_args(PyObject* args) { +flatten_args(PyObject* args) +{ int arg_length = PyTuple_GET_SIZE(args); int total_args = 0; // Get number of args once it's flattened. @@ -171,7 +171,8 @@ flatten_args(PyObject* args) { } static PyObject* -dedup_and_flatten_args(PyObject* args) { +dedup_and_flatten_args(PyObject* args) +{ args = flatten_args(args); int arg_length = PyTuple_GET_SIZE(args); PyObject* temp[arg_length]; @@ -247,18 +248,18 @@ is_unionable(PyObject *obj) } static PyObject * -union_new(PyTypeObject* self, PyObject* param) { +union_new(PyTypeObject* self, PyObject* param) +{ // Check param is a PyType or GenericAlias if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) { - Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; - PyObject *tuple=PyTuple_Pack(2, self_type, param_type); + PyObject *tuple = PyTuple_Pack(2, self_type, param_type); PyObject *newUnionType = Py_Union(tuple); Py_DECREF(param_type); Py_DECREF(self_type); @@ -268,12 +269,13 @@ union_new(PyTypeObject* self, PyObject* param) { static PyObject * -type_or(PyTypeObject* self, PyObject* param) { +type_or(PyTypeObject* self, PyObject* param) +{ return union_new(self, param); } static PyNumberMethods union_as_number = { - .nb_or = (binaryfunc)type_or, // Add __or__ function + .nb_or = (binaryfunc)type_or, // Add __or__ function }; static int @@ -359,7 +361,6 @@ union_repr(PyObject *self) unionobject *alias = (unionobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; _PyUnicodeWriter_Init(&writer); for (Py_ssize_t i = 0; i < len; i++) { @@ -377,10 +378,8 @@ union_repr(PyObject *self) error: _PyUnicodeWriter_Dealloc(&writer); return NULL; - } - PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.Union", @@ -402,8 +401,6 @@ PyTypeObject Py_UnionType = { .tp_repr = union_repr, }; - - PyObject * Py_Union(PyObject *args) { @@ -430,7 +427,7 @@ Py_Union(PyObject *args) } PyObject * -Py_Union_New(PyTypeObject* self, PyObject* param) { - PyObject* new_union = union_new(self, param); - return new_union; +Py_Union_New(PyTypeObject* self, PyObject* param) +{ + return union_new(self, param); } From a1c703b884c3d2753a2d923315df2e67d81b7ba6 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 10:12:49 -0700 Subject: [PATCH 24/60] Implement NE for union richcompare. --- Lib/test/test_types.py | 3 +++ Objects/unionobject.c | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 0fa91937f2282c..12f37d10960d38 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -605,6 +605,7 @@ def test_method_descriptor_types(self): def test_or_types_operator(self): self.assertEqual(int | str, typing.Union[int, str]) + self.assertNotEqual(int | list, typing.Union[int, str]) self.assertEqual(str | int, typing.Union[int, str]) self.assertEqual(int | None, typing.Union[int, None]) self.assertEqual(None | int, typing.Union[int, None]) @@ -643,6 +644,8 @@ def test_or_types_operator(self): 3 | int with self.assertRaises(TypeError): Example() | int + with self.assertRaises(TypeError): + (int | str) < typing.Union[str, int] def test_or_type_operator_with_TypeVar(self): diff --git a/Objects/unionobject.c b/Objects/unionobject.c index b103710a1afa57..9741532f0bb755 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -108,6 +108,12 @@ static PyMethodDef union_methods[] = { static PyObject * union_richcompare(PyObject *a, PyObject *b, int op) { + if (op != Py_EQ && op != Py_NE) { + PyObject *result = Py_NotImplemented; + Py_INCREF(result); + return result; + } + PyObject* b_set = NULL; unionobject *aa = (unionobject *)a; PyObject* a_set = PySet_New(aa->args); @@ -127,7 +133,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) Py_RETURN_FALSE; } - return PyObject_RichCompare(a_set, b_set, Py_EQ); + return PyObject_RichCompare(a_set, b_set, op); } static PyObject* From c4acc73e20a406efd4067b03ec99996b8a8b3ca4 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 10:32:15 -0700 Subject: [PATCH 25/60] Remove is_generic_alias helper method. --- Objects/unionobject.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 9741532f0bb755..7f0ad3a86a934c 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -141,7 +141,7 @@ flatten_args(PyObject* args) { int arg_length = PyTuple_GET_SIZE(args); int total_args = 0; - // Get number of args once it's flattened. + // Get number of total args once it's flattened. for (int i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); @@ -212,19 +212,12 @@ dedup_and_flatten_args(PyObject* args) return new_args; } - static int is_typevar(PyObject *obj) { return is_typing_name(obj, "TypeVar"); } -static int -is_genericalias(PyObject *obj) -{ - return is_typing_name(obj, "GenericAlias"); -} - static int is_new_type(PyObject *obj) { @@ -245,10 +238,10 @@ is_unionable(PyObject *obj) return 1; } return ( - is_genericalias(obj) || is_typevar(obj) || is_new_type(obj) || PyType_Check(obj) || + (PyObject_IsInstance(obj, (PyObject*)&Py_GenericAliasType)) || (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); } From 6a8dfe93d1755d5b60928ac28c5935449bbd6f2d Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 15:49:59 -0700 Subject: [PATCH 26/60] Fix negative ref count. --- Objects/unionobject.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 7f0ad3a86a934c..8e7b89481fd1b4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -96,7 +96,7 @@ is_typing_name(PyObject *obj, char *name) } int res = PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); + // Py_DECREF(module); return res; } @@ -164,15 +164,17 @@ flatten_args(PyObject* args) int nested_arg_length = PyTuple_GET_SIZE(nested_args); for (int j = 0; j < nested_arg_length; j++) { PyObject* nested_arg = PyTuple_GET_ITEM(nested_args, j); + Py_INCREF(nested_arg); PyTuple_SET_ITEM(flattened_args, pos, nested_arg); pos++; } } else { + Py_INCREF(arg); PyTuple_SET_ITEM(flattened_args, pos, arg); pos++; } } - + Py_INCREF(flattened_args); return flattened_args; } @@ -182,6 +184,7 @@ dedup_and_flatten_args(PyObject* args) args = flatten_args(args); int arg_length = PyTuple_GET_SIZE(args); PyObject* temp[arg_length]; + Py_INCREF(temp); for (int i = 0; i < arg_length ; i++) { temp[i] = NULL; } // Add unique elements to an array. @@ -194,6 +197,7 @@ dedup_and_flatten_args(PyObject* args) if (i_tuple == j_tuple) is_duplicate = 1; } if (!is_duplicate) { + Py_INCREF(i_tuple); temp[temp_count] = i_tuple; temp_count++; } @@ -205,10 +209,11 @@ dedup_and_flatten_args(PyObject* args) for(int k = 0; k < arg_length; k++) { temp_arg = temp[k]; - if (temp_arg != NULL) + if (temp_arg != NULL) { PyTuple_SET_ITEM(new_args, k, temp_arg); + } } - + Py_INCREF(new_args); return new_args; } @@ -259,11 +264,9 @@ union_new(PyTypeObject* self, PyObject* param) PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; PyObject *tuple = PyTuple_Pack(2, self_type, param_type); - PyObject *newUnionType = Py_Union(tuple); - Py_DECREF(param_type); - Py_DECREF(self_type); - Py_DECREF(tuple); - return newUnionType; + PyObject *new_union = Py_Union(tuple); + Py_INCREF(new_union); + return new_union; } From af5975181542690e10d42d758c10e51ba8e7fcc2 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 15:52:51 -0700 Subject: [PATCH 27/60] Small code style fixes. --- Objects/unionobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 8e7b89481fd1b4..ba32c09cfdd420 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -96,7 +96,7 @@ is_typing_name(PyObject *obj, char *name) } int res = PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); - // Py_DECREF(module); + Py_DECREF(module); return res; } @@ -184,7 +184,6 @@ dedup_and_flatten_args(PyObject* args) args = flatten_args(args); int arg_length = PyTuple_GET_SIZE(args); PyObject* temp[arg_length]; - Py_INCREF(temp); for (int i = 0; i < arg_length ; i++) { temp[i] = NULL; } // Add unique elements to an array. @@ -209,9 +208,8 @@ dedup_and_flatten_args(PyObject* args) for(int k = 0; k < arg_length; k++) { temp_arg = temp[k]; - if (temp_arg != NULL) { + if (temp_arg != NULL) PyTuple_SET_ITEM(new_args, k, temp_arg); - } } Py_INCREF(new_args); return new_args; From cae1a385cbf80963fa1d50604515e9c285c51dfd Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 16:26:41 -0700 Subject: [PATCH 28/60] Add unionobject to pythoncore.vcxproj.filters --- PCbuild/pythoncore.vcxproj.filters | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1f3883768c9a25..64cf6113f273dc 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1178,10 +1178,13 @@ Objects + + Objects + Resource Files - \ No newline at end of file + From 0cb47c11a6bb425cad5d89eb7604ea40406d2e40 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 27 Jul 2020 16:31:27 -0700 Subject: [PATCH 29/60] Remove semicolon after PyObject_HEAD --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index ba32c09cfdd420..30047f10aea3cf 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -5,7 +5,7 @@ typedef struct { - PyObject_HEAD; + PyObject_HEAD PyObject *args; } unionobject; From 265be78c2ef5471d23d155bd4bf125ce19a0ffea Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 28 Jul 2020 14:00:53 -0700 Subject: [PATCH 30/60] Use _PyTuple_Resize instead of variable length array. --- Objects/unionobject.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 30047f10aea3cf..0252c390b5aea4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -182,9 +182,8 @@ static PyObject* dedup_and_flatten_args(PyObject* args) { args = flatten_args(args); - int arg_length = PyTuple_GET_SIZE(args); - PyObject* temp[arg_length]; - for (int i = 0; i < arg_length ; i++) { temp[i] = NULL; } + Py_ssize_t arg_length = PyTuple_GET_SIZE(args); + PyObject *new_args = PyTuple_New(arg_length); // Add unique elements to an array. int temp_count = 0; @@ -197,20 +196,13 @@ dedup_and_flatten_args(PyObject* args) } if (!is_duplicate) { Py_INCREF(i_tuple); - temp[temp_count] = i_tuple; + PyTuple_SET_ITEM(new_args, temp_count, i_tuple); temp_count++; } } - // Create new tuple from deduplicated array. - PyObject* temp_arg = NULL; - PyObject* new_args = PyTuple_New(temp_count); - for(int k = 0; k < arg_length; k++) { - temp_arg = temp[k]; + _PyTuple_Resize(&new_args, temp_count); - if (temp_arg != NULL) - PyTuple_SET_ITEM(new_args, k, temp_arg); - } Py_INCREF(new_args); return new_args; } From 6fddcd635ef8601f78f7adbad05628206997b9f0 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 28 Jul 2020 14:46:03 -0700 Subject: [PATCH 31/60] Add error handling. --- Objects/unionobject.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 0252c390b5aea4..e5ee559aeb8857 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -129,7 +129,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) b_set = PySet_New(bb->args); } - if (b_set == NULL) { + if (b_set == NULL || a_set == NULL) { Py_RETURN_FALSE; } @@ -154,6 +154,9 @@ flatten_args(PyObject* args) } // Create new tuple of flattened args. PyObject *flattened_args = PyTuple_New(total_args); + if (flattened_args == NULL) { + return NULL; + } int pos = 0; for (int i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); @@ -182,9 +185,14 @@ static PyObject* dedup_and_flatten_args(PyObject* args) { args = flatten_args(args); + if (args == NULL) { + return NULL; + } Py_ssize_t arg_length = PyTuple_GET_SIZE(args); PyObject *new_args = PyTuple_New(arg_length); - + if (new_args == NULL) { + return NULL; + } // Add unique elements to an array. int temp_count = 0; for(int i = 0; i < arg_length; i++) { @@ -202,7 +210,6 @@ dedup_and_flatten_args(PyObject* args) } _PyTuple_Resize(&new_args, temp_count); - Py_INCREF(new_args); return new_args; } @@ -413,6 +420,9 @@ Py_Union(PyObject *args) } PyObject* new_args = dedup_and_flatten_args(args); + if (new_args == NULL) { + return NULL; + } alias->args = new_args; _PyObject_GC_TRACK(alias); return (PyObject *) alias; From 60f023ad3a5bb7a219f03eb9d7debe47b10e995d Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 28 Jul 2020 15:12:25 -0700 Subject: [PATCH 32/60] Add additional tests for union_richcompare. --- Lib/test/test_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 12f37d10960d38..d40f8d66189358 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -646,6 +646,10 @@ def test_or_types_operator(self): Example() | int with self.assertRaises(TypeError): (int | str) < typing.Union[str, int] + with self.assertRaises(TypeError): + (int | str) < (int | bool) + with self.assertRaises(TypeError): + (int | str) <= (int | str) def test_or_type_operator_with_TypeVar(self): From 99ba1fdf20e38a3efc95240605fbdcb318ec6371 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2020 22:43:28 +0000 Subject: [PATCH 33/60] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2020-07-28-22-43-27.bpo-41428.FM6xsI.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-28-22-43-27.bpo-41428.FM6xsI.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-28-22-43-27.bpo-41428.FM6xsI.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-28-22-43-27.bpo-41428.FM6xsI.rst new file mode 100644 index 00000000000000..a6652de9275117 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-28-22-43-27.bpo-41428.FM6xsI.rst @@ -0,0 +1 @@ +Implement PEP 604. This supports (int | str) etc. in place of Union[str, int]. \ No newline at end of file From ad945fd3743c3b6e17f0cdc32adf14ac586305bb Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 30 Jul 2020 11:38:35 -0700 Subject: [PATCH 34/60] Update error check for PySet_New call. --- Objects/unionobject.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index e5ee559aeb8857..a3cbc8740154dd 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -117,6 +117,9 @@ union_richcompare(PyObject *a, PyObject *b, int op) PyObject* b_set = NULL; unionobject *aa = (unionobject *)a; PyObject* a_set = PySet_New(aa->args); + if (a_set == NULL) { + Py_RETURN_FALSE; + } if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); @@ -129,7 +132,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) b_set = PySet_New(bb->args); } - if (b_set == NULL || a_set == NULL) { + if (b_set == NULL) { Py_RETURN_FALSE; } From 0586c41a575cf52f80618c1d5fbc1e494a899a03 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 30 Jul 2020 12:19:15 -0700 Subject: [PATCH 35/60] Allow Any and other _SpecialForm --- Lib/test/test_types.py | 8 ++++++++ Objects/unionobject.c | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index d40f8d66189358..5886db70a812e8 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -694,6 +694,14 @@ def test_or_type_operator_with_NewType(self): def test_or_type_operator_with_IO(self): assert typing.IO | str == typing.Union[typing.IO, str] + def test_or_type_operator_with_SpecialForm(self): + assert typing.Any | str == typing.Union[typing.Any, str] + assert typing.NoReturn | str = typing.Union[typing.NoReturn, str] + assert typing.ClassVar[int] | str = typing.Union[typing.ClassVar[int], str] + assert typing.Optional[int] | str = typing.Union[typing.Optional[int], str] + assert typing.Union[int, bool] | str = typing.Union[typing.Union[int, bool], str] + + class Forward: ... # T = typing.TypeVar('T') # ForwardAfter = T | 'Forward' diff --git a/Objects/unionobject.c b/Objects/unionobject.c index a3cbc8740154dd..21995c90ba02d2 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -223,6 +223,12 @@ is_typevar(PyObject *obj) return is_typing_name(obj, "TypeVar"); } +static int +is_special_form(PyObject *obj) +{ + return is_typing_name(obj, "_SpecialForm"); +} + static int is_new_type(PyObject *obj) { @@ -245,6 +251,7 @@ is_unionable(PyObject *obj) return ( is_typevar(obj) || is_new_type(obj) || + is_special_form(obj) || PyType_Check(obj) || (PyObject_IsInstance(obj, (PyObject*)&Py_GenericAliasType)) || (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || From e44a657eac54f7b26bd1d4a1d08a35666cc5886c Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 30 Jul 2020 12:43:12 -0700 Subject: [PATCH 36/60] Update repr to handle NoneType properly. --- Lib/test/test_types.py | 9 +++++---- Objects/unionobject.c | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 5886db70a812e8..4d14cb4bd6ec32 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -696,11 +696,12 @@ def test_or_type_operator_with_IO(self): def test_or_type_operator_with_SpecialForm(self): assert typing.Any | str == typing.Union[typing.Any, str] - assert typing.NoReturn | str = typing.Union[typing.NoReturn, str] - assert typing.ClassVar[int] | str = typing.Union[typing.ClassVar[int], str] - assert typing.Optional[int] | str = typing.Union[typing.Optional[int], str] - assert typing.Union[int, bool] | str = typing.Union[typing.Union[int, bool], str] + assert typing.NoReturn | str == typing.Union[typing.NoReturn, str] + assert typing.Optional[int] | str == typing.Union[typing.Optional[int], str] + assert typing.Union[int, bool] | str == typing.Union[typing.Union[int, bool], str] + def test_or_type_repr(self): + assert repr(int | None) == "int | None" class Forward: ... # T = typing.TypeVar('T') diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 21995c90ba02d2..384ca6c2c90d65 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -306,6 +306,11 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) goto done; } + if (p == &_PyNone_Type) { + r = PyUnicode_FromString("None"); + goto done; + } + if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { goto done; } From ba3110c83e8b7b225944bfdc7a0a04df2cc3b808 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 30 Jul 2020 14:08:30 -0700 Subject: [PATCH 37/60] Add Union to types.py --- Lib/test/test_types.py | 1 + Lib/types.py | 1 + Lib/typing.py | 7 ++++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 4d14cb4bd6ec32..b6ade6440728f0 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -617,6 +617,7 @@ def test_or_types_operator(self): self.assertEqual(typing.List[int] | None, typing.Union[typing.List[int], None]) self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) self.assertEqual(str | float | int | complex | int, (int | str) | (float | complex)) + self.assertEqual(typing.Union[str, int, typing.List[int]], str | int | typing.List[int]) self.assertEqual( BaseException | bool | diff --git a/Lib/types.py b/Lib/types.py index ad2020ec69b637..9642e7212caac6 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -294,6 +294,7 @@ def wrapped(*args, **kwargs): GenericAlias = type(list[int]) +Union = type(int | str) __all__ = [n for n in globals() if n[:1] != '_'] diff --git a/Lib/typing.py b/Lib/typing.py index 2d2d7e3172280d..7be4b3cb229599 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -116,7 +116,7 @@ # The pseudo-submodules 're' and 'io' are part of the public # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. - +# Union = type(int | str) def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). @@ -145,8 +145,9 @@ def _type_check(arg, msg, is_argument=True): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") - if isinstance(arg, (type, TypeVar, ForwardRef)): + if isinstance(arg, (type, TypeVar, ForwardRef, types.Union)): return arg + if not callable(arg): raise TypeError(f"{msg} Got {arg!r:.100}.") return arg @@ -205,7 +206,7 @@ def _remove_dups_flatten(parameters): # Flatten out Union[Union[...], ...]. params = [] for p in parameters: - if isinstance(p, _UnionGenericAlias): + if isinstance(p, (_UnionGenericAlias, types.Union)): params.extend(p.__args__) elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: params.extend(p[1:]) From e052c53e4ef277dc432a9d32b97e0b7019d861d0 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 31 Jul 2020 12:10:46 -0700 Subject: [PATCH 38/60] Remove commented out code. --- Lib/typing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index 7be4b3cb229599..7a59af0189f7b1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -116,7 +116,6 @@ # The pseudo-submodules 're' and 'io' are part of the public # namespace, but excluded from __all__ because they might stomp on # legitimate imports of those modules. -# Union = type(int | str) def _type_check(arg, msg, is_argument=True): """Check that the argument is a type, and return it (internal helper). From 15e0d2949bfcc0ae68821ccf66e7db160925d9d5 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 31 Jul 2020 12:14:38 -0700 Subject: [PATCH 39/60] Update test with nested unions. --- Lib/test/test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index b6ade6440728f0..0495e22b94351d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -699,7 +699,7 @@ def test_or_type_operator_with_SpecialForm(self): assert typing.Any | str == typing.Union[typing.Any, str] assert typing.NoReturn | str == typing.Union[typing.NoReturn, str] assert typing.Optional[int] | str == typing.Union[typing.Optional[int], str] - assert typing.Union[int, bool] | str == typing.Union[typing.Union[int, bool], str] + assert typing.Union[int, bool] | str == typing.Union[int, bool, str] def test_or_type_repr(self): assert repr(int | None) == "int | None" From 2e31288c93c8d1b5c57955a792e1c7bc455b9b9c Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 31 Jul 2020 12:16:21 -0700 Subject: [PATCH 40/60] Add additional tests for optionals. --- Lib/test/test_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 0495e22b94351d..a2e718d1b0f5af 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -699,6 +699,7 @@ def test_or_type_operator_with_SpecialForm(self): assert typing.Any | str == typing.Union[typing.Any, str] assert typing.NoReturn | str == typing.Union[typing.NoReturn, str] assert typing.Optional[int] | str == typing.Union[typing.Optional[int], str] + assert typing.Optional[int] | str == typing.Union[int, str, None] assert typing.Union[int, bool] | str == typing.Union[int, bool, str] def test_or_type_repr(self): From 2a12a43445fe46d8b7d60dfa996be031d7aec1db Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 31 Jul 2020 12:28:42 -0700 Subject: [PATCH 41/60] Add support for int | int == int --- Lib/test/test_types.py | 1 + Lib/typing.py | 1 - Objects/unionobject.c | 10 +++++----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index a2e718d1b0f5af..4b8fad2fed8571 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -618,6 +618,7 @@ def test_or_types_operator(self): self.assertEqual(None | typing.List[int], typing.Union[None, typing.List[int]]) self.assertEqual(str | float | int | complex | int, (int | str) | (float | complex)) self.assertEqual(typing.Union[str, int, typing.List[int]], str | int | typing.List[int]) + self.assertEqual(int | int, int) self.assertEqual( BaseException | bool | diff --git a/Lib/typing.py b/Lib/typing.py index 7a59af0189f7b1..506434e9c59a7f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -146,7 +146,6 @@ def _type_check(arg, msg, is_argument=True): raise TypeError(f"Plain {arg} is not valid as type argument") if isinstance(arg, (type, TypeVar, ForwardRef, types.Union)): return arg - if not callable(arg): raise TypeError(f"{msg} Got {arg!r:.100}.") return arg diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 384ca6c2c90d65..0713215661eb2d 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -121,21 +121,21 @@ union_richcompare(PyObject *a, PyObject *b, int op) Py_RETURN_FALSE; } + PyTypeObject *type = Py_TYPE(b); if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); b_set = PySet_New(b_args); - } - - PyTypeObject *type = Py_TYPE(b); - if (type == &Py_UnionType) { + } else if (type == &Py_UnionType) { unionobject *bb = (unionobject *)b; b_set = PySet_New(bb->args); + } else { + PyObject * tup = PyTuple_Pack(1, b); + b_set = PySet_New(tup); } if (b_set == NULL) { Py_RETURN_FALSE; } - return PyObject_RichCompare(a_set, b_set, op); } From 7116cd99444cdf5922e741ab8f65914d59f4717c Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 31 Jul 2020 16:13:55 -0700 Subject: [PATCH 42/60] Add more error handling to unionobject --- Objects/unionobject.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 0713215661eb2d..61f1743e516502 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -271,7 +271,13 @@ union_new(PyTypeObject* self, PyObject* param) PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; PyObject *tuple = PyTuple_Pack(2, self_type, param_type); + if (tuple == NULL) { + return NULL; + } PyObject *new_union = Py_Union(tuple); + if (new_union == NULL) { + return NULL; + } Py_INCREF(new_union); return new_union; } From 2c0befb6b49941b216efc44599523c3bb1c8e0aa Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Mon, 3 Aug 2020 15:11:46 -0700 Subject: [PATCH 43/60] Handle NoneType vs. None --- Objects/unionobject.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 61f1743e516502..00b91f7280498e 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -56,6 +56,9 @@ union_instancecheck(PyObject *self, PyObject *instance) Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (arg == Py_None) { + arg = (PyObject *)Py_TYPE(arg); + } if (PyType_Check(arg)) { if (PyObject_IsInstance(instance, arg) != 0) { @@ -124,6 +127,13 @@ union_richcompare(PyObject *a, PyObject *b, int op) PyTypeObject *type = Py_TYPE(b); if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); + int b_arg_length = PyTuple_GET_SIZE(b_args); + for (int i = 0; i < b_arg_length; i++) { + PyObject* arg = PyTuple_GET_ITEM(b_args, i); + if (arg == (PyObject *)&_PyNone_Type) { + PyTuple_SET_ITEM(b_args, i, Py_None); + } + } b_set = PySet_New(b_args); } else if (type == &Py_UnionType) { unionobject *bb = (unionobject *)b; @@ -268,9 +278,7 @@ union_new(PyTypeObject* self, PyObject* param) return Py_NotImplemented; } - PyObject *param_type = param == Py_None ? (PyObject *)Py_TYPE(param) : param; - PyTypeObject *self_type = (PyObject *)self == Py_None ? Py_TYPE(self) : self; - PyObject *tuple = PyTuple_Pack(2, self_type, param_type); + PyObject *tuple = PyTuple_Pack(2, self, param); if (tuple == NULL) { return NULL; } @@ -312,11 +320,6 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) goto done; } - if (p == &_PyNone_Type) { - r = PyUnicode_FromString("None"); - goto done; - } - if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { goto done; } From 8d59e44f5fe4984f19c8a222dfddab60ed3887f7 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 4 Aug 2020 13:46:02 -0700 Subject: [PATCH 44/60] Don't allow generics in isinstance calls. --- Lib/test/test_isinstance.py | 6 ++++++ Objects/unionobject.c | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 4a53b2de33d156..6d89448151cd31 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -218,6 +218,12 @@ def test_isinstance_with_or_union(self): self.assertTrue(isinstance(2, typing.List | int)) self.assertFalse(isinstance(2, typing.List | typing.Tuple)) self.assertTrue(isinstance(None, int | None)) + with self.assertRaises(TypeError): + isinstance(2, list[int]) + with self.assertRaises(TypeError): + isinstance(2, list[int] | int) + + def test_subclass_normal(self): # normal classes diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 00b91f7280498e..61c6610d3cdfa9 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -56,14 +56,19 @@ union_instancecheck(PyObject *self, PyObject *instance) Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + PyTypeObject* arg_type = Py_TYPE(arg); if (arg == Py_None) { - arg = (PyObject *)Py_TYPE(arg); + arg = (PyObject *)arg_type; } if (PyType_Check(arg)) { if (PyObject_IsInstance(instance, arg) != 0) { return self; } + } else if (arg_type == &Py_GenericAliasType) { + PyErr_SetString(PyExc_TypeError, + "isinstance() argument 2 cannot contain a parameterized generic"); + return NULL; } } return instance; From 9f014e204ed955b90e48bc58ede59842d0d90563 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 5 Aug 2020 11:00:47 -0700 Subject: [PATCH 45/60] Add errors for union subclass. --- Lib/test/test_isinstance.py | 4 ++++ Objects/unionobject.c | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 6d89448151cd31..b79fa35d64af35 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -273,6 +273,10 @@ def test_isinstance_recursion_limit(self): def test_subclass_with_union(self): self.assertTrue(issubclass(int, int | float | int)) self.assertTrue(issubclass(str, str | Child | str)) + with self.assertRaises(TypeError): + issubclass(2, Child | Super) + with self.assertRaises(TypeError): + issubclass(int, list[int] | Child) def test_issubclass_refcount_handling(self): # bpo-39382: abstract_issubclass() didn't hold item reference while diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 61c6610d3cdfa9..f4f7b927d90448 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -77,15 +77,24 @@ union_instancecheck(PyObject *self, PyObject *instance) static PyObject * union_subclasscheck(PyObject *self, PyObject *instance) { - unionobject *alias = (unionobject *) self; + if (!PyType_Check(instance)) { + PyErr_SetString(PyExc_TypeError, "issubclass() arg 1 must be a class"); + return NULL; + } + unionobject *alias = (unionobject *)self; Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + if (PyType_Check(arg)) { if (PyType_IsSubtype((PyTypeObject *)instance, (PyTypeObject *)arg) != 0) { return self; } + } else if (Py_TYPE(arg) == &Py_GenericAliasType) { + PyErr_SetString(PyExc_TypeError, + "issubclass() argument 2 cannot contain a parameterized generic"); + return NULL; } } return instance; From 30bb872a70f77805fa6d0672a2ce0518e579f3ca Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 12 Aug 2020 11:50:13 -0700 Subject: [PATCH 46/60] Implement code review suggestions. - Reorder checking valid arguments for isinstance and issubclass methods. - Rename variables as suggested. - Combine checks, where possible. --- Lib/test/test_isinstance.py | 3 + Objects/unionobject.c | 122 +++++++++++++++++++----------------- 2 files changed, 68 insertions(+), 57 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index b79fa35d64af35..ab9f0d835459a6 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -218,10 +218,13 @@ def test_isinstance_with_or_union(self): self.assertTrue(isinstance(2, typing.List | int)) self.assertFalse(isinstance(2, typing.List | typing.Tuple)) self.assertTrue(isinstance(None, int | None)) + self.assertFalse(isinstance(3.14, int | str)) with self.assertRaises(TypeError): isinstance(2, list[int]) with self.assertRaises(TypeError): isinstance(2, list[int] | int) + with self.assertRaises(TypeError): + isinstance(2, int | str | list[int] | float) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index f4f7b927d90448..5255088e66aba4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -43,10 +43,16 @@ static PyMemberDef union_members[] = { {0} }; -static PyObject * -union_getattro(PyObject *self, PyObject *name) -{ - return PyObject_GenericGetAttr(self, name); +static int* +check_args(PyObject *args) { + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(args, iarg); + if (Py_TYPE(arg) == &Py_GenericAliasType) { + return 0; + } + } + return 1; } static PyObject * @@ -54,26 +60,25 @@ union_instancecheck(PyObject *self, PyObject *instance) { unionobject *alias = (unionobject *) self; Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + if (!check_args(alias->args)) { + PyErr_SetString(PyExc_TypeError, + "isinstance() argument 2 cannot contain a parameterized generic"); + return NULL; + } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - PyTypeObject* arg_type = Py_TYPE(arg); + PyTypeObject *arg_type = Py_TYPE(arg); if (arg == Py_None) { arg = (PyObject *)arg_type; } - if (PyType_Check(arg)) { - if (PyObject_IsInstance(instance, arg) != 0) - { - return self; - } - } else if (arg_type == &Py_GenericAliasType) { - PyErr_SetString(PyExc_TypeError, - "isinstance() argument 2 cannot contain a parameterized generic"); - return NULL; + if (PyType_Check(arg) && PyObject_IsInstance(instance, arg) != 0) { + Py_RETURN_TRUE; } } - return instance; + Py_RETURN_FALSE; } + static PyObject * union_subclasscheck(PyObject *self, PyObject *instance) { @@ -82,22 +87,19 @@ union_subclasscheck(PyObject *self, PyObject *instance) return NULL; } unionobject *alias = (unionobject *)self; + if (!check_args(alias->args)) { + PyErr_SetString(PyExc_TypeError, + "issubclass() argument 2 cannot contain a parameterized generic"); + return NULL; + } Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - - if (PyType_Check(arg)) { - if (PyType_IsSubtype((PyTypeObject *)instance, (PyTypeObject *)arg) != 0) - { - return self; - } - } else if (Py_TYPE(arg) == &Py_GenericAliasType) { - PyErr_SetString(PyExc_TypeError, - "issubclass() argument 2 cannot contain a parameterized generic"); - return NULL; + if (PyType_Check(arg) && (PyType_IsSubtype((PyTypeObject *)instance, (PyTypeObject *)arg) != 0)) { + Py_RETURN_TRUE; } } - return instance; + Py_RETURN_FALSE; } static int @@ -142,7 +144,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); int b_arg_length = PyTuple_GET_SIZE(b_args); - for (int i = 0; i < b_arg_length; i++) { + for (Py_ssize_t i = 0; i < b_arg_length; i++) { PyObject* arg = PyTuple_GET_ITEM(b_args, i); if (arg == (PyObject *)&_PyNone_Type) { PyTuple_SET_ITEM(b_args, i, Py_None); @@ -152,14 +154,21 @@ union_richcompare(PyObject *a, PyObject *b, int op) } else if (type == &Py_UnionType) { unionobject *bb = (unionobject *)b; b_set = PySet_New(bb->args); + if (b_set == NULL) { + return NULL; + } } else { - PyObject * tup = PyTuple_Pack(1, b); - b_set = PySet_New(tup); + PyObject *tuple = PyTuple_Pack(1, b); + if (tuple == NULL ) { + return NULL; + } + b_set = PySet_New(tuple); + Py_DECREF(tuple); + if (b_set == NULL) { + return NULL; + } } - if (b_set == NULL) { - Py_RETURN_FALSE; - } return PyObject_RichCompare(a_set, b_set, op); } @@ -169,7 +178,7 @@ flatten_args(PyObject* args) int arg_length = PyTuple_GET_SIZE(args); int total_args = 0; // Get number of total args once it's flattened. - for (int i = 0; i < arg_length; i++) { + for (Py_ssize_t i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); if (arg_type == &Py_UnionType) { @@ -185,7 +194,7 @@ flatten_args(PyObject* args) return NULL; } int pos = 0; - for (int i = 0; i < arg_length; i++) { + for (Py_ssize_t i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); if (arg_type == &Py_UnionType) { @@ -204,7 +213,6 @@ flatten_args(PyObject* args) pos++; } } - Py_INCREF(flattened_args); return flattened_args; } @@ -221,23 +229,24 @@ dedup_and_flatten_args(PyObject* args) return NULL; } // Add unique elements to an array. - int temp_count = 0; - for(int i = 0; i < arg_length; i++) { + int added_items = 0; + for(Py_ssize_t i = 0; i < arg_length; i++) { int is_duplicate = 0; PyObject* i_tuple = PyTuple_GET_ITEM(args, i); for(int j = i + 1; j < arg_length; j++) { PyObject* j_tuple = PyTuple_GET_ITEM(args, j); - if (i_tuple == j_tuple) is_duplicate = 1; + if (i_tuple == j_tuple) { + is_duplicate = 1; + } } if (!is_duplicate) { Py_INCREF(i_tuple); - PyTuple_SET_ITEM(new_args, temp_count, i_tuple); - temp_count++; + PyTuple_SET_ITEM(new_args, added_items, i_tuple); + added_items++; } } - - _PyTuple_Resize(&new_args, temp_count); - Py_INCREF(new_args); + Py_DECREF(args); + _PyTuple_Resize(&new_args, added_items); return new_args; } @@ -297,10 +306,10 @@ union_new(PyTypeObject* self, PyObject* param) return NULL; } PyObject *new_union = Py_Union(tuple); + Py_DECREF(tuple); if (new_union == NULL) { return NULL; } - Py_INCREF(new_union); return new_union; } @@ -331,16 +340,16 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) if (p == Py_Ellipsis) { // The Ellipsis object r = PyUnicode_FromString("..."); - goto done; + goto exit; } if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { - goto done; + goto exit; } if (tmp != NULL) { Py_DECREF(tmp); if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) { - goto done; + goto exit; } if (tmp != NULL) { Py_DECREF(tmp); @@ -350,13 +359,13 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) } if (_PyObject_LookupAttrId(p, &PyId___qualname__, &qualname) < 0) { - goto done; + goto exit; } if (qualname == NULL) { goto use_repr; } if (_PyObject_LookupAttrId(p, &PyId___module__, &module) < 0) { - goto done; + goto exit; } if (module == NULL || module == Py_None) { goto use_repr; @@ -368,17 +377,17 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) { // builtins don't need a module name r = PyObject_Str(qualname); - goto done; + goto exit; } else { r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto done; + goto exit; } use_repr: r = PyObject_Repr(p); -done: +exit: Py_XDECREF(qualname); Py_XDECREF(module); if (r == NULL) { @@ -401,10 +410,8 @@ union_repr(PyObject *self) _PyUnicodeWriter writer; _PyUnicodeWriter_Init(&writer); for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { - goto error; - } + if (i > 0 && _PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { + goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); if (union_repr_item(&writer, p) < 0) { @@ -430,7 +437,7 @@ PyTypeObject Py_UnionType = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_hash = union_hash, .tp_traverse = union_traverse, - .tp_getattro = union_getattro, + .tp_getattro = PyObject_GenericGetAttr, .tp_members = union_members, .tp_methods = union_methods, .tp_richcompare = union_richcompare, @@ -458,6 +465,7 @@ Py_Union(PyObject *args) } PyObject* new_args = dedup_and_flatten_args(args); + Py_DECREF(args); if (new_args == NULL) { return NULL; } From 6fe4956a2ebeeec6739373bb38d4e0a7ab100946 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 12 Aug 2020 11:57:14 -0700 Subject: [PATCH 47/60] Increase code sharing between is_typing_name and is_new_type --- Objects/unionobject.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 5255088e66aba4..07d6f23819a78b 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -102,6 +102,16 @@ union_subclasscheck(PyObject *self, PyObject *instance) Py_RETURN_FALSE; } +static int +is_typing_module(PyObject *obj) { + PyObject *module = PyObject_GetAttrString(obj, "__module__"); + if (module == NULL) { + return -1; + } + Py_DECREF(module); + return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); +} + static int is_typing_name(PyObject *obj, char *name) { @@ -109,14 +119,7 @@ is_typing_name(PyObject *obj, char *name) if (strcmp(type->tp_name, name) != 0) { return 0; } - PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); - if (module == NULL) { - return -1; - } - int res = PyUnicode_Check(module) - && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - return res; + return is_typing_module(obj); } static PyMethodDef union_methods[] = { @@ -268,11 +271,7 @@ is_new_type(PyObject *obj) if (!PyObject_IsInstance(obj, (PyObject *)&PyFunction_Type)) { return 0; } - PyObject *module = PyObject_GetAttrString(obj, "__module__"); - if (module == NULL) { - return 0; - } - return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); + return is_typing_module(obj); } static int From bae3d2a956609800eec8c7f713e6e48ae99a159d Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 12 Aug 2020 14:05:53 -0700 Subject: [PATCH 48/60] Add tests, fix reference leaks. --- Lib/test/test_isinstance.py | 2 ++ Objects/unionobject.c | 33 +++++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index ab9f0d835459a6..91e79c295481db 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -276,6 +276,8 @@ def test_isinstance_recursion_limit(self): def test_subclass_with_union(self): self.assertTrue(issubclass(int, int | float | int)) self.assertTrue(issubclass(str, str | Child | str)) + self.assertFalse(issubclass(dict, float|str)) + self.assertFalse(issubclass(object, float|str)) with self.assertRaises(TypeError): issubclass(2, Child | Super) with self.assertRaises(TypeError): diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 07d6f23819a78b..c3d41f0e634d69 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -146,6 +146,9 @@ union_richcompare(PyObject *a, PyObject *b, int op) PyTypeObject *type = Py_TYPE(b); if (is_typing_name(b, "_UnionGenericAlias")) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); + if (b_args == NULL) { + return NULL; + } int b_arg_length = PyTuple_GET_SIZE(b_args); for (Py_ssize_t i = 0; i < b_arg_length; i++) { PyObject* arg = PyTuple_GET_ITEM(b_args, i); @@ -154,25 +157,31 @@ union_richcompare(PyObject *a, PyObject *b, int op) } } b_set = PySet_New(b_args); + Py_DECREF(b_args); } else if (type == &Py_UnionType) { unionobject *bb = (unionobject *)b; b_set = PySet_New(bb->args); if (b_set == NULL) { + Py_DECREF(a_set); return NULL; } } else { PyObject *tuple = PyTuple_Pack(1, b); if (tuple == NULL ) { + Py_DECREF(a_set); return NULL; } b_set = PySet_New(tuple); Py_DECREF(tuple); - if (b_set == NULL) { - return NULL; - } } - - return PyObject_RichCompare(a_set, b_set, op); + if (b_set == NULL) { + Py_DECREF(a_set); + return NULL; + } + PyObject *result = PyObject_RichCompare(a_set, b_set, op); + Py_DECREF(a_set); + Py_DECREF(b_set); + return result; } static PyObject* @@ -268,7 +277,8 @@ is_special_form(PyObject *obj) static int is_new_type(PyObject *obj) { - if (!PyObject_IsInstance(obj, (PyObject *)&PyFunction_Type)) { + PyTypeObject *type = Py_TYPE(obj); + if (type != &PyFunction_Type) { return 0; } return is_typing_module(obj); @@ -280,21 +290,24 @@ is_unionable(PyObject *obj) if (obj == Py_None) { return 1; } + PyTypeObject *type = Py_TYPE(obj); return ( is_typevar(obj) || is_new_type(obj) || is_special_form(obj) || PyType_Check(obj) || - (PyObject_IsInstance(obj, (PyObject*)&Py_GenericAliasType)) || - (PyObject_IsInstance(obj, (PyObject *)&PyType_Type) == 1) || - (PyObject_IsInstance(obj, (PyObject *)&Py_UnionType) == 1)); + type == &Py_GenericAliasType || + type == &Py_UnionType); } static PyObject * union_new(PyTypeObject* self, PyObject* param) { + if (param == NULL) { + return NULL; + } // Check param is a PyType or GenericAlias - if ((param == NULL) || !is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) + if (!is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; From 17a7277897810197e2a2b70ce39c566d550bd3a2 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 12 Aug 2020 14:30:27 -0700 Subject: [PATCH 49/60] Update tp_flags for unionobject. --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index c3d41f0e634d69..bfd23f1f68726b 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -446,7 +446,7 @@ PyTypeObject Py_UnionType = { .tp_dealloc = unionobject_dealloc, .tp_alloc = PyType_GenericAlloc, .tp_free = PyObject_GC_Del, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT, .tp_hash = union_hash, .tp_traverse = union_traverse, .tp_getattro = PyObject_GenericGetAttr, From f47bce09bc067cb7e1ad029a1d59ff2588b6ea56 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 13 Aug 2020 08:57:27 -0700 Subject: [PATCH 50/60] Return int. --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index c3d41f0e634d69..69af0e155a2994 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -43,7 +43,7 @@ static PyMemberDef union_members[] = { {0} }; -static int* +static int check_args(PyObject *args) { Py_ssize_t nargs = PyTuple_GET_SIZE(args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { From 0735352cacb31604246288a95f5e6229dc7a62a2 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Thu, 13 Aug 2020 09:22:37 -0700 Subject: [PATCH 51/60] Fix NoneType comparison side effect. --- Objects/unionobject.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 69af0e155a2994..aab18f58eb0740 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -153,10 +153,25 @@ union_richcompare(PyObject *a, PyObject *b, int op) for (Py_ssize_t i = 0; i < b_arg_length; i++) { PyObject* arg = PyTuple_GET_ITEM(b_args, i); if (arg == (PyObject *)&_PyNone_Type) { - PyTuple_SET_ITEM(b_args, i, Py_None); + arg = Py_None; + } + if (b_set == NULL) { + PyObject* args = PyTuple_Pack(1, arg); + if (args == NULL) { + return NULL; + } + b_set = PySet_New(args); + Py_DECREF(args); + if (b_set == NULL) { + return NULL; + } + } else { + int err = PySet_Add(b_set, arg); + if (err == -1) { + return NULL; + } } } - b_set = PySet_New(b_args); Py_DECREF(b_args); } else if (type == &Py_UnionType) { unionobject *bb = (unionobject *)b; From e9e5a5e03833f38ec40ad8e391bb35291ad68b7e Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Fri, 14 Aug 2020 11:58:52 -0700 Subject: [PATCH 52/60] Remove GC flag from unionobject. --- Objects/unionobject.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 77f11e510925de..4df99592685e70 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -14,7 +14,6 @@ unionobject_dealloc(PyObject *self) { unionobject *alias = (unionobject *)self; - _PyObject_GC_UNTRACK(self); Py_XDECREF(alias->args); self->ob_type->tp_free(self); } @@ -30,14 +29,6 @@ union_hash(PyObject *self) return h1; } -static int -union_traverse(PyObject *self, visitproc visit, void *arg) -{ - unionobject *alias = (unionobject *)self; - Py_VISIT(alias->args); - return 0; -} - static PyMemberDef union_members[] = { {"__args__", T_OBJECT, offsetof(unionobject, args), READONLY}, {0} @@ -460,10 +451,9 @@ PyTypeObject Py_UnionType = { .tp_basicsize = sizeof(unionobject), .tp_dealloc = unionobject_dealloc, .tp_alloc = PyType_GenericAlloc, - .tp_free = PyObject_GC_Del, + .tp_free = PyObject_Del, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_hash = union_hash, - .tp_traverse = union_traverse, .tp_getattro = PyObject_GenericGetAttr, .tp_members = union_members, .tp_methods = union_methods, @@ -485,7 +475,7 @@ Py_Union(PyObject *args) Py_INCREF(args); } - unionobject *alias = PyObject_GC_New(unionobject, &Py_UnionType); + unionobject *alias = PyObject_New(unionobject, &Py_UnionType); if (alias == NULL) { Py_DECREF(args); return NULL; @@ -497,7 +487,6 @@ Py_Union(PyObject *args) return NULL; } alias->args = new_args; - _PyObject_GC_TRACK(alias); return (PyObject *) alias; } From 385a338c728e3acd122dded83bcf01c9e60d8b78 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 18 Aug 2020 12:57:11 -0700 Subject: [PATCH 53/60] Implement code review suggestions - Use goto target to reduce code repetition. - Fix style nits. - Rename method check_args -> is_generic_alias_in_args. - Fix potential reference leaks. - Check for error result from is_unionable. - Small refactor to repr method --- Objects/unionobject.c | 63 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 4df99592685e70..1299713b622bd4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -35,7 +35,7 @@ static PyMemberDef union_members[] = { }; static int -check_args(PyObject *args) { +is_generic_alias_in_args(PyObject *args) { Py_ssize_t nargs = PyTuple_GET_SIZE(args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); @@ -51,14 +51,14 @@ union_instancecheck(PyObject *self, PyObject *instance) { unionobject *alias = (unionobject *) self; Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); - if (!check_args(alias->args)) { + if (!is_generic_alias_in_args(alias->args)) { PyErr_SetString(PyExc_TypeError, "isinstance() argument 2 cannot contain a parameterized generic"); return NULL; } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - PyTypeObject *arg_type = Py_TYPE(arg); + PyTypeObject *arg_type = Py_TYPE(arg); if (arg == Py_None) { arg = (PyObject *)arg_type; } @@ -78,7 +78,7 @@ union_subclasscheck(PyObject *self, PyObject *instance) return NULL; } unionobject *alias = (unionobject *)self; - if (!check_args(alias->args)) { + if (!is_generic_alias_in_args(alias->args)) { PyErr_SetString(PyExc_TypeError, "issubclass() argument 2 cannot contain a parameterized generic"); return NULL; @@ -99,8 +99,9 @@ is_typing_module(PyObject *obj) { if (module == NULL) { return -1; } + int is_typing = PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); Py_DECREF(module); - return PyUnicode_Check(module) && _PyUnicode_EqualToASCIIString(module, "typing"); + return is_typing; } static int @@ -131,14 +132,18 @@ union_richcompare(PyObject *a, PyObject *b, int op) unionobject *aa = (unionobject *)a; PyObject* a_set = PySet_New(aa->args); if (a_set == NULL) { - Py_RETURN_FALSE; + return NULL; } PyTypeObject *type = Py_TYPE(b); - if (is_typing_name(b, "_UnionGenericAlias")) { + int is_typing_union = is_typing_name(b, "_UnionGenericAlias"); + if (is_typing_union < 0) { + goto error; + } + if (is_typing_union) { PyObject* b_args = PyObject_GetAttrString(b, "__args__"); if (b_args == NULL) { - return NULL; + goto error; } int b_arg_length = PyTuple_GET_SIZE(b_args); for (Py_ssize_t i = 0; i < b_arg_length; i++) { @@ -149,17 +154,17 @@ union_richcompare(PyObject *a, PyObject *b, int op) if (b_set == NULL) { PyObject* args = PyTuple_Pack(1, arg); if (args == NULL) { - return NULL; + goto error; } b_set = PySet_New(args); Py_DECREF(args); if (b_set == NULL) { - return NULL; + goto error; } } else { int err = PySet_Add(b_set, arg); if (err == -1) { - return NULL; + goto error; } } } @@ -168,26 +173,26 @@ union_richcompare(PyObject *a, PyObject *b, int op) unionobject *bb = (unionobject *)b; b_set = PySet_New(bb->args); if (b_set == NULL) { - Py_DECREF(a_set); - return NULL; + goto error; } } else { PyObject *tuple = PyTuple_Pack(1, b); if (tuple == NULL ) { - Py_DECREF(a_set); - return NULL; + goto error; } b_set = PySet_New(tuple); Py_DECREF(tuple); } if (b_set == NULL) { - Py_DECREF(a_set); - return NULL; + goto error; } PyObject *result = PyObject_RichCompare(a_set, b_set, op); Py_DECREF(a_set); Py_DECREF(b_set); return result; +error: + Py_DECREF(a_set); + return NULL; } static PyObject* @@ -313,7 +318,15 @@ union_new(PyTypeObject* self, PyObject* param) return NULL; } // Check param is a PyType or GenericAlias - if (!is_unionable((PyObject *)param) || !is_unionable((PyObject*)self)) + int is_param_unionable = is_unionable((PyObject *)param); + if (is_param_unionable < 0) { + return NULL; + } + int is_self_unionable = is_unionable((PyObject*)self); + if (is_self_unionable < 0) { + return NULL; + } + if (!is_param_unionable || !is_self_unionable) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; @@ -352,7 +365,6 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) PyObject *qualname = NULL; PyObject *module = NULL; PyObject *r = NULL; - PyObject *tmp; int err; if (p == Py_Ellipsis) { @@ -360,17 +372,16 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) r = PyUnicode_FromString("..."); goto exit; } - - if (_PyObject_LookupAttrId(p, &PyId___origin__, &tmp) < 0) { + int has_origin = _PyObject_HasAttrId(p, &PyId___origin__); + if (has_origin < 0) { goto exit; } - if (tmp != NULL) { - Py_DECREF(tmp); - if (_PyObject_LookupAttrId(p, &PyId___args__, &tmp) < 0) { + if (has_origin) { + int has_args = _PyObject_HasAttrId(p, &PyId___args__); + if (has_args < 0) { goto exit; } - if (tmp != NULL) { - Py_DECREF(tmp); + if (has_args) { // It looks like a GenericAlias goto use_repr; } From dddc598f859caa6a5d08ee4b2e50331ce44af8eb Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Tue, 18 Aug 2020 14:40:49 -0700 Subject: [PATCH 54/60] Refactor and remove PyUnion_New --- Include/unionobject.h | 1 - Objects/typeobject.c | 8 ++++- Objects/unionobject.c | 71 ++++++++++++++++++------------------------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/Include/unionobject.h b/Include/unionobject.h index 27642373d04f49..d72c3e88e0ca67 100644 --- a/Include/unionobject.h +++ b/Include/unionobject.h @@ -5,7 +5,6 @@ extern "C" { #endif PyAPI_FUNC(PyObject *) Py_Union(PyObject *); -PyAPI_FUNC(PyObject *) Py_Union_New(PyTypeObject *, PyObject *); PyAPI_DATA(PyTypeObject) Py_UnionType; #ifdef __cplusplus diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4291abe4323b3a..591f1952b092f7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3743,7 +3743,13 @@ type_is_gc(PyTypeObject *type) static PyObject * type_or(PyTypeObject* self, PyObject* param) { - return Py_Union_New(self, param); + PyObject *tuple = PyTuple_Pack(2, self, param); + if (tuple == NULL) { + return NULL; + } + PyObject *new_union = Py_Union(tuple); + Py_DECREF(tuple); + return new_union; } static PyNumberMethods type_as_number = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 1299713b622bd4..5c37736d05cb33 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -69,7 +69,6 @@ union_instancecheck(PyObject *self, PyObject *instance) Py_RETURN_FALSE; } - static PyObject * union_subclasscheck(PyObject *self, PyObject *instance) { @@ -182,9 +181,9 @@ union_richcompare(PyObject *a, PyObject *b, int op) } b_set = PySet_New(tuple); Py_DECREF(tuple); - } - if (b_set == NULL) { - goto error; + if (b_set == NULL) { + goto error; + } } PyObject *result = PyObject_RichCompare(a_set, b_set, op); Py_DECREF(a_set); @@ -192,6 +191,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) return result; error: Py_DECREF(a_set); + Py_DECREF(b_set); return NULL; } @@ -312,45 +312,17 @@ is_unionable(PyObject *obj) } static PyObject * -union_new(PyTypeObject* self, PyObject* param) +type_or(PyTypeObject* self, PyObject* param) { - if (param == NULL) { - return NULL; - } - // Check param is a PyType or GenericAlias - int is_param_unionable = is_unionable((PyObject *)param); - if (is_param_unionable < 0) { - return NULL; - } - int is_self_unionable = is_unionable((PyObject*)self); - if (is_self_unionable < 0) { - return NULL; - } - if (!is_param_unionable || !is_self_unionable) - { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - PyObject *tuple = PyTuple_Pack(2, self, param); if (tuple == NULL) { return NULL; } PyObject *new_union = Py_Union(tuple); Py_DECREF(tuple); - if (new_union == NULL) { - return NULL; - } return new_union; } - -static PyObject * -type_or(PyTypeObject* self, PyObject* param) -{ - return union_new(self, param); -} - static PyNumberMethods union_as_number = { .nb_or = (binaryfunc)type_or, // Add __or__ function }; @@ -486,23 +458,38 @@ Py_Union(PyObject *args) Py_INCREF(args); } + // Check arguments are unionable. + int nargs = PyTuple_GET_SIZE(args); + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + PyObject *arg = PyTuple_GET_ITEM(args, iarg); + if (arg == NULL) { + goto error; + } + int is_arg_unionable = is_unionable(arg); + if (is_arg_unionable < 0) { + goto error; + } + if (!is_arg_unionable) { + Py_DECREF(args); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + } + unionobject *alias = PyObject_New(unionobject, &Py_UnionType); if (alias == NULL) { - Py_DECREF(args); - return NULL; + goto error; } PyObject* new_args = dedup_and_flatten_args(args); - Py_DECREF(args); if (new_args == NULL) { - return NULL; + goto error; } + Py_DECREF(args); alias->args = new_args; return (PyObject *) alias; -} -PyObject * -Py_Union_New(PyTypeObject* self, PyObject* param) -{ - return union_new(self, param); +error: + Py_DECREF(args); + return NULL; } From 8c06c1d509a52b6d114ca09433e6a7b6b2216e20 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 9 Sep 2020 00:48:42 +0100 Subject: [PATCH 55/60] General cleanup, fix reference leaks, add new test for possible crashes --- Lib/test/test_types.py | 6 +- Objects/unionobject.c | 182 ++++++++++++++++++----------------------- 2 files changed, 83 insertions(+), 105 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 4b8fad2fed8571..040554d967e7d7 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -652,7 +652,11 @@ def test_or_types_operator(self): (int | str) < (int | bool) with self.assertRaises(TypeError): (int | str) <= (int | str) - + with self.assertRaises(TypeError): + # Check that we don't crash if typing.Union does not have a tuple in __args__ + x = typing.Union[str, int] + x.__args__ = [str, int] + (int | str ) == x def test_or_type_operator_with_TypeVar(self): TV = typing.TypeVar('T') diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 5c37736d05cb33..f1c91cc15cfcd3 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -29,11 +29,6 @@ union_hash(PyObject *self) return h1; } -static PyMemberDef union_members[] = { - {"__args__", T_OBJECT, offsetof(unionobject, args), READONLY}, - {0} -}; - static int is_generic_alias_in_args(PyObject *args) { Py_ssize_t nargs = PyTuple_GET_SIZE(args); @@ -58,9 +53,8 @@ union_instancecheck(PyObject *self, PyObject *instance) } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); - PyTypeObject *arg_type = Py_TYPE(arg); if (arg == Py_None) { - arg = (PyObject *)arg_type; + arg = (PyObject *)&_PyNone_Type; } if (PyType_Check(arg) && PyObject_IsInstance(instance, arg) != 0) { Py_RETURN_TRUE; @@ -113,86 +107,73 @@ is_typing_name(PyObject *obj, char *name) return is_typing_module(obj); } -static PyMethodDef union_methods[] = { - {"__instancecheck__", union_instancecheck, METH_O}, - {"__subclasscheck__", union_subclasscheck, METH_O}, - {0}}; - static PyObject * union_richcompare(PyObject *a, PyObject *b, int op) { + PyObject *result = NULL; if (op != Py_EQ && op != Py_NE) { - PyObject *result = Py_NotImplemented; + result = Py_NotImplemented; Py_INCREF(result); return result; } - PyObject* b_set = NULL; - unionobject *aa = (unionobject *)a; - PyObject* a_set = PySet_New(aa->args); + PyTypeObject *type = Py_TYPE(b); + + PyObject* a_set = PySet_New(((unionobject*)a)->args); if (a_set == NULL) { return NULL; } + PyObject* b_set = PySet_New(NULL); + if (b_set == NULL) { + goto exit; + } - PyTypeObject *type = Py_TYPE(b); + // Populate b_set with the data from the right object int is_typing_union = is_typing_name(b, "_UnionGenericAlias"); if (is_typing_union < 0) { - goto error; + goto exit; } if (is_typing_union) { - PyObject* b_args = PyObject_GetAttrString(b, "__args__"); + PyObject *b_args = PyObject_GetAttrString(b, "__args__"); if (b_args == NULL) { - goto error; + goto exit; + } + if (!PyTuple_CheckExact(b_args)) { + Py_DECREF(b_args); + PyErr_SetString(PyExc_TypeError, "__args__ argument of typing.Union object is not a tuple"); + goto exit; } - int b_arg_length = PyTuple_GET_SIZE(b_args); + Py_ssize_t b_arg_length = PyTuple_GET_SIZE(b_args); for (Py_ssize_t i = 0; i < b_arg_length; i++) { PyObject* arg = PyTuple_GET_ITEM(b_args, i); if (arg == (PyObject *)&_PyNone_Type) { arg = Py_None; } - if (b_set == NULL) { - PyObject* args = PyTuple_Pack(1, arg); - if (args == NULL) { - goto error; - } - b_set = PySet_New(args); - Py_DECREF(args); - if (b_set == NULL) { - goto error; - } - } else { - int err = PySet_Add(b_set, arg); - if (err == -1) { - goto error; - } + if (PySet_Add(b_set, arg) == -1) { + Py_DECREF(b_args); + goto exit; } } Py_DECREF(b_args); } else if (type == &Py_UnionType) { - unionobject *bb = (unionobject *)b; - b_set = PySet_New(bb->args); - if (b_set == NULL) { - goto error; + PyObject* args = ((unionobject*) b)->args; + Py_ssize_t arg_length = PyTuple_GET_SIZE(args); + for (Py_ssize_t i = 0; i < arg_length; i++) { + PyObject* arg = PyTuple_GET_ITEM(args, i); + if (PySet_Add(b_set, arg) == -1) { + goto exit; + } } } else { - PyObject *tuple = PyTuple_Pack(1, b); - if (tuple == NULL ) { - goto error; - } - b_set = PySet_New(tuple); - Py_DECREF(tuple); - if (b_set == NULL) { - goto error; + if (PySet_Add(b_set, b) == -1) { + goto exit; } } - PyObject *result = PyObject_RichCompare(a_set, b_set, op); - Py_DECREF(a_set); - Py_DECREF(b_set); + result = PyObject_RichCompare(a_set, b_set, op); +exit: + Py_XDECREF(a_set); + Py_XDECREF(b_set); return result; -error: - Py_DECREF(a_set); - Py_DECREF(b_set); - return NULL; } static PyObject* @@ -205,8 +186,7 @@ flatten_args(PyObject* args) PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); if (arg_type == &Py_UnionType) { - unionobject *alias = (unionobject *)arg; - total_args = total_args + PyTuple_GET_SIZE(alias->args); + total_args += PyTuple_GET_SIZE(((unionobject*) arg)->args); } else { total_args++; } @@ -216,13 +196,12 @@ flatten_args(PyObject* args) if (flattened_args == NULL) { return NULL; } - int pos = 0; + Py_ssize_t pos = 0; for (Py_ssize_t i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); if (arg_type == &Py_UnionType) { - unionobject *alias = (unionobject *)arg; - PyObject* nested_args = alias->args; + PyObject* nested_args = ((unionobject*)arg)->args; int nested_arg_length = PyTuple_GET_SIZE(nested_args); for (int j = 0; j < nested_arg_length; j++) { PyObject* nested_arg = PyTuple_GET_ITEM(nested_args, j); @@ -253,18 +232,18 @@ dedup_and_flatten_args(PyObject* args) } // Add unique elements to an array. int added_items = 0; - for(Py_ssize_t i = 0; i < arg_length; i++) { + for (Py_ssize_t i = 0; i < arg_length; i++) { int is_duplicate = 0; - PyObject* i_tuple = PyTuple_GET_ITEM(args, i); - for(int j = i + 1; j < arg_length; j++) { - PyObject* j_tuple = PyTuple_GET_ITEM(args, j); - if (i_tuple == j_tuple) { + PyObject* i_element = PyTuple_GET_ITEM(args, i); + for (Py_ssize_t j = i + 1; j < arg_length; j++) { + PyObject* j_element = PyTuple_GET_ITEM(args, j); + if (i_element == j_element) { is_duplicate = 1; } } if (!is_duplicate) { - Py_INCREF(i_tuple); - PyTuple_SET_ITEM(new_args, added_items, i_tuple); + Py_INCREF(i_element); + PyTuple_SET_ITEM(new_args, added_items, i_element); added_items++; } } @@ -323,10 +302,6 @@ type_or(PyTypeObject* self, PyObject* param) return new_union; } -static PyNumberMethods union_as_number = { - .nb_or = (binaryfunc)type_or, // Add __or__ function -}; - static int union_repr_item(_PyUnicodeWriter *writer, PyObject *p) { @@ -348,6 +323,7 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) if (has_origin < 0) { goto exit; } + if (has_origin) { int has_args = _PyObject_HasAttrId(p, &PyId___args__); if (has_args < 0) { @@ -387,18 +363,14 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) use_repr: r = PyObject_Repr(p); - exit: Py_XDECREF(qualname); Py_XDECREF(module); if (r == NULL) { - // error if any of the above PyObject_Repr/PyUnicode_From* fail - err = -1; - } - else { - err = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); + return -1; } + err = _PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); return err; } @@ -425,6 +397,20 @@ union_repr(PyObject *self) return NULL; } +static PyMemberDef union_members[] = { + {"__args__", T_OBJECT, offsetof(unionobject, args), READONLY}, + {0} +}; + +static PyMethodDef union_methods[] = { + {"__instancecheck__", union_instancecheck, METH_O}, + {"__subclasscheck__", union_subclasscheck, METH_O}, + {0}}; + +static PyNumberMethods union_as_number = { + .nb_or = (binaryfunc)type_or, // Add __or__ function +}; + PyTypeObject Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.Union", @@ -448,48 +434,36 @@ PyTypeObject Py_UnionType = { PyObject * Py_Union(PyObject *args) { - if (!PyTuple_Check(args)) { - args = PyTuple_Pack(1, args); - if (args == NULL) { - return NULL; - } - } - else { - Py_INCREF(args); - } + assert(PyTuple_CheckExact(args)); + + unionobject* result = NULL; // Check arguments are unionable. int nargs = PyTuple_GET_SIZE(args); for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); if (arg == NULL) { - goto error; + return NULL; } int is_arg_unionable = is_unionable(arg); if (is_arg_unionable < 0) { - goto error; + return NULL; } if (!is_arg_unionable) { - Py_DECREF(args); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } } - unionobject *alias = PyObject_New(unionobject, &Py_UnionType); - if (alias == NULL) { - goto error; + result = PyObject_New(unionobject, &Py_UnionType); + if (result == NULL) { + return NULL; } - PyObject* new_args = dedup_and_flatten_args(args); - if (new_args == NULL) { - goto error; + result->args = dedup_and_flatten_args(args); + if (result->args == NULL) { + Py_DECREF(result); + return NULL; } - Py_DECREF(args); - alias->args = new_args; - return (PyObject *) alias; - -error: - Py_DECREF(args); - return NULL; -} + return (PyObject*)result; +} \ No newline at end of file From 37699f1bdd1bec79137f9aeb0413cb61384b3e58 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 9 Sep 2020 00:52:35 +0100 Subject: [PATCH 56/60] Add newline --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index f1c91cc15cfcd3..b86a83e1fd6df7 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -466,4 +466,4 @@ Py_Union(PyObject *args) return NULL; } return (PyObject*)result; -} \ No newline at end of file +} From 47a4cb3ea9ec097faa6f3382fd3b067939842338 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 9 Sep 2020 02:19:54 +0100 Subject: [PATCH 57/60] Make unionobject APIs private --- Include/Python.h | 1 - Include/internal/pycore_unionobject.h | 17 +++++++++++++++++ Include/unionobject.h | 13 ------------- Makefile.pre.in | 1 + Objects/abstract.c | 3 ++- Objects/typeobject.c | 3 ++- Objects/unionobject.c | 18 +++++++++--------- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 9 files changed, 35 insertions(+), 25 deletions(-) create mode 100644 Include/internal/pycore_unionobject.h delete mode 100644 Include/unionobject.h diff --git a/Include/Python.h b/Include/Python.h index 3bfd491f6106f4..57f71d41d8d477 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -123,7 +123,6 @@ #include "genobject.h" #include "descrobject.h" #include "genericaliasobject.h" -#include "unionobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h new file mode 100644 index 00000000000000..fa8ba6ed944c1a --- /dev/null +++ b/Include/internal/pycore_unionobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_UNIONOBJECT_H +#define Py_INTERNAL_UNIONOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_FUNC(PyObject *) _Py_Union(PyObject *args); +PyAPI_DATA(PyTypeObject) _Py_UnionType; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIONOBJECT_H */ diff --git a/Include/unionobject.h b/Include/unionobject.h deleted file mode 100644 index d72c3e88e0ca67..00000000000000 --- a/Include/unionobject.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef Py_UNIONTYPEOBJECT_H -#define Py_UNIONTYPEOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_FUNC(PyObject *) Py_Union(PyObject *); -PyAPI_DATA(PyTypeObject) Py_UnionType; - -#ifdef __cplusplus -} -#endif -#endif /* !Py_UNIONTYPEOBJECT_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index fc204f53b5e122..370a2a73202990 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1123,6 +1123,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_sysmodule.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_unionobject.h \ $(srcdir)/Include/internal/pycore_warnings.h \ $(DTRACE_HEADERS) diff --git a/Objects/abstract.c b/Objects/abstract.c index 3744bf67517180..9313d286bfc901 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1,6 +1,7 @@ /* Abstract Object Interface (many thanks to Jim Fulton) */ #include "Python.h" +#include "pycore_unionobject.h" // _Py_UnionType && _Py_Union() #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -2604,7 +2605,7 @@ recursive_issubclass(PyObject *derived, PyObject *cls) return -1; PyTypeObject *type = Py_TYPE(cls); - int is_union = (PyType_Check(type) && type == &Py_UnionType); + int is_union = (PyType_Check(type) && type == &_Py_UnionType); if (!is_union && !check_class(cls, "issubclass() arg 2 must be a class," " a tuple of classes, or a union.")) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 591f1952b092f7..bc8fcbeb92f25a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,6 +6,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_unionobject.h" // _Py_Union() #include "frameobject.h" #include "structmember.h" // PyMemberDef @@ -3747,7 +3748,7 @@ type_or(PyTypeObject* self, PyObject* param) { if (tuple == NULL) { return NULL; } - PyObject *new_union = Py_Union(tuple); + PyObject *new_union = _Py_Union(tuple); Py_DECREF(tuple); return new_union; } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index b86a83e1fd6df7..32326c7ad79c87 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -1,6 +1,6 @@ // types.Union -- used to represent e.g. Union[int, str], int | str #include "Python.h" -#include "pycore_object.h" +#include "pycore_unionobject.h" #include "structmember.h" @@ -155,7 +155,7 @@ union_richcompare(PyObject *a, PyObject *b, int op) } } Py_DECREF(b_args); - } else if (type == &Py_UnionType) { + } else if (type == &_Py_UnionType) { PyObject* args = ((unionobject*) b)->args; Py_ssize_t arg_length = PyTuple_GET_SIZE(args); for (Py_ssize_t i = 0; i < arg_length; i++) { @@ -185,7 +185,7 @@ flatten_args(PyObject* args) for (Py_ssize_t i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); - if (arg_type == &Py_UnionType) { + if (arg_type == &_Py_UnionType) { total_args += PyTuple_GET_SIZE(((unionobject*) arg)->args); } else { total_args++; @@ -200,7 +200,7 @@ flatten_args(PyObject* args) for (Py_ssize_t i = 0; i < arg_length; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); PyTypeObject* arg_type = Py_TYPE(arg); - if (arg_type == &Py_UnionType) { + if (arg_type == &_Py_UnionType) { PyObject* nested_args = ((unionobject*)arg)->args; int nested_arg_length = PyTuple_GET_SIZE(nested_args); for (int j = 0; j < nested_arg_length; j++) { @@ -287,7 +287,7 @@ is_unionable(PyObject *obj) is_special_form(obj) || PyType_Check(obj) || type == &Py_GenericAliasType || - type == &Py_UnionType); + type == &_Py_UnionType); } static PyObject * @@ -297,7 +297,7 @@ type_or(PyTypeObject* self, PyObject* param) if (tuple == NULL) { return NULL; } - PyObject *new_union = Py_Union(tuple); + PyObject *new_union = _Py_Union(tuple); Py_DECREF(tuple); return new_union; } @@ -411,7 +411,7 @@ static PyNumberMethods union_as_number = { .nb_or = (binaryfunc)type_or, // Add __or__ function }; -PyTypeObject Py_UnionType = { +PyTypeObject _Py_UnionType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.Union", .tp_doc = "Represent a PEP 604 union type\n" @@ -432,7 +432,7 @@ PyTypeObject Py_UnionType = { }; PyObject * -Py_Union(PyObject *args) +_Py_Union(PyObject *args) { assert(PyTuple_CheckExact(args)); @@ -455,7 +455,7 @@ Py_Union(PyObject *args) } } - result = PyObject_New(unionobject, &Py_UnionType); + result = PyObject_New(unionobject, &_Py_UnionType); if (result == NULL) { return NULL; } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 104956d265cd17..70cb32fa530496 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -196,6 +196,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 64cf6113f273dc..748bebe051b3ca 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -573,6 +573,9 @@ Include\internal + + Include\internal + Modules\zlib From 7f89abdf2530232184f87fc8c570b366b7194b09 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 9 Sep 2020 03:03:48 +0100 Subject: [PATCH 58/60] Add minor test for GenericAlias --- Lib/test/test_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 040554d967e7d7..ba4e408e0cd288 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -709,6 +709,7 @@ def test_or_type_operator_with_SpecialForm(self): def test_or_type_repr(self): assert repr(int | None) == "int | None" + assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]" class Forward: ... # T = typing.TypeVar('T') From d0be56fdb4fbcff077e80c25be1db086c8ffa144 Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 9 Sep 2020 10:58:53 -0700 Subject: [PATCH 59/60] Remove ellipsis from repr. --- Objects/unionobject.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 32326c7ad79c87..0ef7abb4c55c28 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -314,11 +314,6 @@ union_repr_item(_PyUnicodeWriter *writer, PyObject *p) PyObject *r = NULL; int err; - if (p == Py_Ellipsis) { - // The Ellipsis object - r = PyUnicode_FromString("..."); - goto exit; - } int has_origin = _PyObject_HasAttrId(p, &PyId___origin__); if (has_origin < 0) { goto exit; From 59f06f71227eb7e90c53a3067e7dd36aed481c7b Mon Sep 17 00:00:00 2001 From: Maggie Moss Date: Wed, 9 Sep 2020 11:32:28 -0700 Subject: [PATCH 60/60] Remove commented out test code. --- Lib/test/test_types.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index ba4e408e0cd288..f499fb9c8c51a4 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -15,6 +15,8 @@ class Example: pass +class Forward: ... + class TypesTests(unittest.TestCase): def test_truth_values(self): @@ -711,13 +713,6 @@ def test_or_type_repr(self): assert repr(int | None) == "int | None" assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]" -class Forward: ... -# T = typing.TypeVar('T') -# ForwardAfter = T | 'Forward' -# ForwardBefore = 'Forward' | T -# def forward_after(x: ForwardAfter[int]) -> None: ... -# def forward_before(x: ForwardBefore[int]) -> None: ... -# class Forward: ... class MappingProxyTests(unittest.TestCase): mappingproxy = types.MappingProxyType