diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 66ccf3c96a65a4..d7b277e9eae03e 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -357,11 +357,26 @@ Other objects .. versionadded:: 3.3 -``(items)`` (:class:`tuple`) [*matching-items*] - The object must be a Python sequence whose length is the number of format units +``(items)`` (sequence) [*matching-items*] + The object must be a Python sequence (except :class:`str`, :class:`bytes` + or :class:`bytearray`) whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in *items*. Format units for sequences may be nested. + If *items* contains format units which store a :ref:`borrowed buffer + ` (``s``, ``s#``, ``z``, ``z#``, ``y``, or ``y#``) + or a :term:`borrowed reference` (``S``, ``Y``, ``U``, ``O``, or ``O!``), + the object must be a Python tuple. + The *converter* for the ``O&`` format unit in *items* must not store + a borrowed buffer or a borrowed reference. + + .. versionchanged:: next + :class:`str` and :class:`bytearray` objects no longer accepted as a sequence. + + .. deprecated:: next + Non-tuple sequences are deprecated if *items* contains format units + which store a borrowed buffer or a borrowed reference. + A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ec0d050c84c6f8..9765bd31333c3c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1938,6 +1938,13 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) +* Non-tuple sequences are deprecated as argument for the ``(items)`` + format unit in :c:func:`PyArg_ParseTuple` and other + :ref:`argument parsing ` functions if *items* contains + format units which store a :ref:`borrowed buffer ` + or a :term:`borrowed reference`. + (Contributed by Serhiy Storchaka in :gh:`50333`.) + * The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. Use :c:func:`PySequence_Contains` instead. (Contributed by Yuki Kobayashi in :gh:`127896`.) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3a32967e721903..ffa9b5009c96df 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1383,7 +1383,7 @@ _PyOpcode_macro_expansion[256] = { [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, - [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_RANGE] = { .nuops = 2, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_NEXT_RANGE, OPARG_REPLACED, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_range.h b/Include/internal/pycore_range.h index bf045ec4fd8332..ab8324cf524269 100644 --- a/Include/internal/pycore_range.h +++ b/Include/internal/pycore_range.h @@ -10,11 +10,26 @@ extern "C" { typedef struct { PyObject_HEAD - long start; - long step; long len; + long end; + long step; + long stop; } _PyRangeIterObject; +static inline long +_PyRangeIter_GetLengthAndStart(_PyRangeIterObject *r, long *value) +{ + long len = FT_ATOMIC_LOAD_LONG_RELAXED(r->len); + *value = r->end - r->step * len; + return len; +} + +static inline void +_PyRangeIter_SetLength(_PyRangeIterObject *r, long len) +{ + FT_ATOMIC_STORE_LONG_RELAXED(r->len, len); +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 2b79fffc14bc33..686cbc6dd20058 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -128,22 +128,21 @@ extern "C" { #define _GUARD_NOS_INT 377 #define _GUARD_NOS_UNICODE 378 #define _GUARD_NOT_EXHAUSTED_LIST 379 -#define _GUARD_NOT_EXHAUSTED_RANGE 380 -#define _GUARD_NOT_EXHAUSTED_TUPLE 381 -#define _GUARD_TOS_FLOAT 382 -#define _GUARD_TOS_INT 383 -#define _GUARD_TOS_UNICODE 384 -#define _GUARD_TYPE_VERSION 385 -#define _GUARD_TYPE_VERSION_AND_LOCK 386 +#define _GUARD_NOT_EXHAUSTED_TUPLE 380 +#define _GUARD_TOS_FLOAT 381 +#define _GUARD_TOS_INT 382 +#define _GUARD_TOS_UNICODE 383 +#define _GUARD_TYPE_VERSION 384 +#define _GUARD_TYPE_VERSION_AND_LOCK 385 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 387 -#define _INIT_CALL_PY_EXACT_ARGS 388 -#define _INIT_CALL_PY_EXACT_ARGS_0 389 -#define _INIT_CALL_PY_EXACT_ARGS_1 390 -#define _INIT_CALL_PY_EXACT_ARGS_2 391 -#define _INIT_CALL_PY_EXACT_ARGS_3 392 -#define _INIT_CALL_PY_EXACT_ARGS_4 393 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 386 +#define _INIT_CALL_PY_EXACT_ARGS 387 +#define _INIT_CALL_PY_EXACT_ARGS_0 388 +#define _INIT_CALL_PY_EXACT_ARGS_1 389 +#define _INIT_CALL_PY_EXACT_ARGS_2 390 +#define _INIT_CALL_PY_EXACT_ARGS_3 391 +#define _INIT_CALL_PY_EXACT_ARGS_4 392 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -153,163 +152,163 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 394 +#define _IS_NONE 393 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 395 -#define _ITER_CHECK_RANGE 396 -#define _ITER_CHECK_TUPLE 397 -#define _ITER_JUMP_LIST 398 -#define _ITER_JUMP_RANGE 399 -#define _ITER_JUMP_TUPLE 400 -#define _ITER_NEXT_LIST 401 -#define _ITER_NEXT_LIST_TIER_TWO 402 -#define _ITER_NEXT_RANGE 403 -#define _ITER_NEXT_TUPLE 404 -#define _JUMP_TO_TOP 405 +#define _ITER_CHECK_LIST 394 +#define _ITER_CHECK_RANGE 395 +#define _ITER_CHECK_TUPLE 396 +#define _ITER_JUMP_LIST 397 +#define _ITER_JUMP_TUPLE 398 +#define _ITER_NEXT_LIST 399 +#define _ITER_NEXT_LIST_TIER_TWO 400 +#define _ITER_NEXT_RANGE 401 +#define _ITER_NEXT_RANGE_TIER_TWO 402 +#define _ITER_NEXT_TUPLE 403 +#define _JUMP_TO_TOP 404 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 406 -#define _LOAD_ATTR_CLASS 407 +#define _LOAD_ATTR 405 +#define _LOAD_ATTR_CLASS 406 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 408 -#define _LOAD_ATTR_METHOD_LAZY_DICT 409 -#define _LOAD_ATTR_METHOD_NO_DICT 410 -#define _LOAD_ATTR_METHOD_WITH_VALUES 411 -#define _LOAD_ATTR_MODULE 412 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 413 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 414 -#define _LOAD_ATTR_PROPERTY_FRAME 415 -#define _LOAD_ATTR_SLOT 416 -#define _LOAD_ATTR_WITH_HINT 417 +#define _LOAD_ATTR_INSTANCE_VALUE 407 +#define _LOAD_ATTR_METHOD_LAZY_DICT 408 +#define _LOAD_ATTR_METHOD_NO_DICT 409 +#define _LOAD_ATTR_METHOD_WITH_VALUES 410 +#define _LOAD_ATTR_MODULE 411 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 412 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 413 +#define _LOAD_ATTR_PROPERTY_FRAME 414 +#define _LOAD_ATTR_SLOT 415 +#define _LOAD_ATTR_WITH_HINT 416 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 418 +#define _LOAD_BYTECODE 417 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 419 -#define _LOAD_CONST_INLINE_BORROW 420 +#define _LOAD_CONST_INLINE 418 +#define _LOAD_CONST_INLINE_BORROW 419 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 421 -#define _LOAD_FAST_0 422 -#define _LOAD_FAST_1 423 -#define _LOAD_FAST_2 424 -#define _LOAD_FAST_3 425 -#define _LOAD_FAST_4 426 -#define _LOAD_FAST_5 427 -#define _LOAD_FAST_6 428 -#define _LOAD_FAST_7 429 +#define _LOAD_FAST 420 +#define _LOAD_FAST_0 421 +#define _LOAD_FAST_1 422 +#define _LOAD_FAST_2 423 +#define _LOAD_FAST_3 424 +#define _LOAD_FAST_4 425 +#define _LOAD_FAST_5 426 +#define _LOAD_FAST_6 427 +#define _LOAD_FAST_7 428 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 430 -#define _LOAD_FAST_BORROW_0 431 -#define _LOAD_FAST_BORROW_1 432 -#define _LOAD_FAST_BORROW_2 433 -#define _LOAD_FAST_BORROW_3 434 -#define _LOAD_FAST_BORROW_4 435 -#define _LOAD_FAST_BORROW_5 436 -#define _LOAD_FAST_BORROW_6 437 -#define _LOAD_FAST_BORROW_7 438 +#define _LOAD_FAST_BORROW 429 +#define _LOAD_FAST_BORROW_0 430 +#define _LOAD_FAST_BORROW_1 431 +#define _LOAD_FAST_BORROW_2 432 +#define _LOAD_FAST_BORROW_3 433 +#define _LOAD_FAST_BORROW_4 434 +#define _LOAD_FAST_BORROW_5 435 +#define _LOAD_FAST_BORROW_6 436 +#define _LOAD_FAST_BORROW_7 437 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 439 -#define _LOAD_GLOBAL_BUILTINS 440 -#define _LOAD_GLOBAL_MODULE 441 +#define _LOAD_GLOBAL 438 +#define _LOAD_GLOBAL_BUILTINS 439 +#define _LOAD_GLOBAL_MODULE 440 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 442 -#define _LOAD_SMALL_INT_0 443 -#define _LOAD_SMALL_INT_1 444 -#define _LOAD_SMALL_INT_2 445 -#define _LOAD_SMALL_INT_3 446 +#define _LOAD_SMALL_INT 441 +#define _LOAD_SMALL_INT_0 442 +#define _LOAD_SMALL_INT_1 443 +#define _LOAD_SMALL_INT_2 444 +#define _LOAD_SMALL_INT_3 445 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 447 +#define _MAKE_CALLARGS_A_TUPLE 446 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 448 +#define _MAKE_WARM 447 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 449 -#define _MAYBE_EXPAND_METHOD_KW 450 -#define _MONITOR_CALL 451 -#define _MONITOR_CALL_KW 452 -#define _MONITOR_JUMP_BACKWARD 453 -#define _MONITOR_RESUME 454 +#define _MAYBE_EXPAND_METHOD 448 +#define _MAYBE_EXPAND_METHOD_KW 449 +#define _MONITOR_CALL 450 +#define _MONITOR_CALL_KW 451 +#define _MONITOR_JUMP_BACKWARD 452 +#define _MONITOR_RESUME 453 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 455 -#define _POP_JUMP_IF_TRUE 456 +#define _POP_JUMP_IF_FALSE 454 +#define _POP_JUMP_IF_TRUE 455 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 457 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 458 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 459 +#define _POP_TOP_LOAD_CONST_INLINE 456 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 457 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 458 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 460 +#define _PUSH_FRAME 459 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 461 -#define _PY_FRAME_GENERAL 462 -#define _PY_FRAME_KW 463 -#define _QUICKEN_RESUME 464 -#define _REPLACE_WITH_TRUE 465 +#define _PUSH_NULL_CONDITIONAL 460 +#define _PY_FRAME_GENERAL 461 +#define _PY_FRAME_KW 462 +#define _QUICKEN_RESUME 463 +#define _REPLACE_WITH_TRUE 464 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 466 -#define _SEND 467 -#define _SEND_GEN_FRAME 468 +#define _SAVE_RETURN_OFFSET 465 +#define _SEND 466 +#define _SEND_GEN_FRAME 467 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 469 -#define _STORE_ATTR 470 -#define _STORE_ATTR_INSTANCE_VALUE 471 -#define _STORE_ATTR_SLOT 472 -#define _STORE_ATTR_WITH_HINT 473 +#define _START_EXECUTOR 468 +#define _STORE_ATTR 469 +#define _STORE_ATTR_INSTANCE_VALUE 470 +#define _STORE_ATTR_SLOT 471 +#define _STORE_ATTR_WITH_HINT 472 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 474 -#define _STORE_FAST_0 475 -#define _STORE_FAST_1 476 -#define _STORE_FAST_2 477 -#define _STORE_FAST_3 478 -#define _STORE_FAST_4 479 -#define _STORE_FAST_5 480 -#define _STORE_FAST_6 481 -#define _STORE_FAST_7 482 +#define _STORE_FAST 473 +#define _STORE_FAST_0 474 +#define _STORE_FAST_1 475 +#define _STORE_FAST_2 476 +#define _STORE_FAST_3 477 +#define _STORE_FAST_4 478 +#define _STORE_FAST_5 479 +#define _STORE_FAST_6 480 +#define _STORE_FAST_7 481 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 483 -#define _STORE_SUBSCR 484 +#define _STORE_SLICE 482 +#define _STORE_SUBSCR 483 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _STORE_SUBSCR_LIST_INT 485 +#define _STORE_SUBSCR_LIST_INT 484 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 486 -#define _TO_BOOL 487 +#define _TIER2_RESUME_CHECK 485 +#define _TO_BOOL 486 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 488 +#define _TO_BOOL_STR 487 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 489 +#define _UNPACK_SEQUENCE 488 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 489 +#define MAX_UOP_ID 488 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c089328f2e967d..2be1816adc41d0 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -199,8 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, - [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, - [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, + [_ITER_NEXT_RANGE_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -415,7 +414,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_NOS_INT] = "_GUARD_NOS_INT", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", - [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", @@ -437,7 +435,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", - [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_TIER_TWO] = "_ITER_NEXT_RANGE_TIER_TWO", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", @@ -936,9 +934,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _ITER_CHECK_RANGE: return 0; - case _GUARD_NOT_EXHAUSTED_RANGE: - return 0; - case _ITER_NEXT_RANGE: + case _ITER_NEXT_RANGE_TIER_TWO: return 0; case _FOR_ITER_GEN_FRAME: return 0; diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 05d25a8d510bf7..60822d5d794a18 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -63,6 +63,9 @@ NULL = None +class CustomError(Exception): + pass + class Index: def __index__(self): return 99 @@ -586,13 +589,13 @@ def test_tuple(self): ret = getargs_tuple(1, (2, 3)) self.assertEqual(ret, (1,2,3)) - # make sure invalid tuple arguments are handled correctly - class seq: + # make sure invalid sequence arguments are handled correctly + class TestSeq: def __len__(self): return 2 def __getitem__(self, n): - raise ValueError - self.assertRaises(TypeError, getargs_tuple, 1, seq()) + raise CustomError + self.assertRaises(CustomError, getargs_tuple, 1, TestSeq()) class Keywords_TestCase(unittest.TestCase): def test_kwargs(self): @@ -1320,33 +1323,69 @@ def test_nonascii_keywords(self): f"this function got an unexpected keyword argument '{name2}'"): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) - def test_nested_tuple(self): + def test_nested_sequence(self): parse = _testcapi.parse_tuple_and_keywords self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), (1, 2, 3, 4)) parse(((1, 2, 3),), {}, '(iii)', ['a']) + parse(([1, 2, 3],), {}, '(iii)', ['a']) with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 2, not 3"): + "argument 1 must be tuple of length 2, not 3"): parse(((1, 2, 3),), {}, '(ii)', ['a']) with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 2, not 1"): + "argument 1 must be tuple of length 2, not 1"): parse(((1,),), {}, '(ii)', ['a']) with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item sequence, not int"): + "argument 1 must be sequence of length 2, not 3"): + parse(([1, 2, 3],), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 1"): + parse(([1,],), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not int"): parse((1,), {}, '(ii)', ['a']) with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item sequence, not bytes"): + "argument 1 must be 2-item tuple, not None$"): + parse((None,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not str"): + parse(('ab',), {}, '(CC)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not bytes"): parse((b'ab',), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not bytearray"): + parse((bytearray(b'ab'),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not dict"): + parse(({},), {}, '(ii)', ['a']) + + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 3-item tuple, not list"): + self.assertEqual(parse(([1, 2, 3],), {}, '(OOO)', ['a']), (1, 2, 3)) + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 2-item tuple, not list"): + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 3"): + parse(([1, 2, 3],), {}, '(OO)', ['a']) + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 2-item tuple, not list"): + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 1"): + parse(([1,],), {}, '(OO)', ['a']) for f in 'es', 'et', 'es#', 'et#': with self.assertRaises(LookupError): # empty encoding "" parse((('a',),), {}, '(' + f + ')', ['a']) with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 1, not 0"): + "argument 1 must be tuple of length 1, not 0"): parse(((),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 1, not 0"): + parse(([],), {}, '(' + f + ')', ['a']) @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') def test_gh_119213(self): diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a006..962fc9d1933eaf 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -1,3 +1,4 @@ +import sys import threading import unittest from test import support @@ -12,8 +13,8 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 - NUMTHREADS = 5 + NUMITEMS = 50000 + NUMTHREADS = 3 NUMMUTATORS = 2 class ContendedTupleIterationTest(unittest.TestCase): @@ -23,7 +24,7 @@ def make_testdata(self, n): def assert_iterator_results(self, results, expected): # Most iterators are not atomic (yet?) so they can skip or duplicate # items, but they should not invent new items (like the range - # iterator currently does). + # iterator has done in the past). extra_items = set(results) - set(expected) self.assertEqual(set(), extra_items) @@ -39,7 +40,7 @@ def test_iteration(self): """Test iteration over a shared container""" seq = self.make_testdata(NUMITEMS) results = [] - start = threading.Barrier(NUMTHREADS) + start = threading.Event() def worker(): idx = 0 start.wait() @@ -47,6 +48,7 @@ def worker(): idx += 1 results.append(idx) threads = self.run_threads(worker) + start.set() for t in threads: t.join() # Each thread has its own iterator, so results should be entirely predictable. @@ -57,7 +59,7 @@ def test_shared_iterator(self): seq = self.make_testdata(NUMITEMS) it = iter(seq) results = [] - start = threading.Barrier(NUMTHREADS) + start = threading.Event() def worker(): items = [] start.wait() @@ -66,9 +68,10 @@ def worker(): items.append(item) results.extend(items) threads = self.run_threads(worker) + start.set() for t in threads: t.join() - self.assert_iterator_results(results, seq) + self.assert_iterator_results(sorted(results), seq) class ContendedListIterationTest(ContendedTupleIterationTest): def make_testdata(self, n): @@ -78,7 +81,7 @@ def test_iteration_while_mutating(self): """Test iteration over a shared mutating container.""" seq = self.make_testdata(NUMITEMS) results = [] - start = threading.Barrier(NUMTHREADS + NUMMUTATORS) + start = threading.Event() endmutate = threading.Event() def mutator(): orig = seq[:] @@ -87,10 +90,7 @@ def mutator(): replacement = (orig * 3)[NUMITEMS//2:] start.wait() while not endmutate.is_set(): - seq.extend(replacement) - seq[:0] = orig - seq.__imul__(2) - seq.extend(seq) + seq[:] = replacement seq[:] = orig def worker(): items = [] @@ -103,6 +103,7 @@ def worker(): try: threads = self.run_threads(worker) mutators = self.run_threads(mutator, numthreads=NUMMUTATORS) + start.set() for t in threads: t.join() finally: @@ -116,12 +117,7 @@ class ContendedRangeIterationTest(ContendedTupleIterationTest): def make_testdata(self, n): return range(n) - def assert_iterator_results(self, results, expected): - # Range iterators that are shared between threads will (right now) - # sometimes produce items after the end of the range, sometimes - # _far_ after the end of the range. That should be fixed, but for - # now, let's just check they're integers that could have resulted - # from stepping beyond the range bounds. - extra_items = set(results) - set(expected) - for item in extra_items: - self.assertEqual((item - expected.start) % expected.step, 0) + +class ContendedLongRangeIterationTest(ContendedTupleIterationTest): + def make_testdata(self, n): + return range(0, sys.maxsize*n, sys.maxsize) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 4843a9e11931b1..38825666c8a2ff 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1734,7 +1734,7 @@ def delx(self): del self.__x # PyCapsule check(_datetime.datetime_CAPI, size('6P')) # rangeiterator - check(iter(range(1)), size('3l')) + check(iter(range(1)), size('4l')) check(iter(range(2**65)), size('3P')) # reverse check(reversed(''), size('nP')) diff --git a/Misc/NEWS.d/next/C_API/2024-12-31-15-28-14.gh-issue-50333.KxQUXa.rst b/Misc/NEWS.d/next/C_API/2024-12-31-15-28-14.gh-issue-50333.KxQUXa.rst new file mode 100644 index 00000000000000..5b761d1d1cf0fc --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-31-15-28-14.gh-issue-50333.KxQUXa.rst @@ -0,0 +1,5 @@ +Non-tuple sequences are deprecated as argument for the ``(items)`` format +unit in :c:func:`PyArg_ParseTuple` and other :ref:`argument parsing +` functions if *items* contains format units which store +a :ref:`borrowed buffer ` or +a :term:`borrowed reference`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-22-20-23.gh-issue-129068.aDFAHD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-22-20-23.gh-issue-129068.aDFAHD.rst new file mode 100644 index 00000000000000..6b709e467465c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-22-20-23.gh-issue-129068.aDFAHD.rst @@ -0,0 +1 @@ +Make range iterator safe in :term:`free-threaded ` build. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a6435e..b5f449d49efe8c 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1,13 +1,15 @@ /* Range object implementation */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_freelist.h" -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_range.h" -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_tuple.h" // _PyTuple_ITEMS() /* Support objects whose length is > PY_SSIZE_T_MAX. @@ -831,11 +833,11 @@ static PyObject * rangeiter_next(PyObject *op) { _PyRangeIterObject *r = (_PyRangeIterObject*)op; - if (r->len > 0) { - long result = r->start; - r->start = result + r->step; - r->len--; - return PyLong_FromLong(result); + long start; + long len = _PyRangeIter_GetLengthAndStart(r, &start); + if (len > 0) { + _PyRangeIter_SetLength(r, len - 1); + return PyLong_FromLong(start); } return NULL; } @@ -844,7 +846,9 @@ static PyObject * rangeiter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { _PyRangeIterObject *r = (_PyRangeIterObject*)op; - return PyLong_FromLong(r->len); + long start; + long len = _PyRangeIter_GetLengthAndStart(r, &start); + return PyLong_FromLong(len); } PyDoc_STRVAR(length_hint_doc, @@ -858,17 +862,19 @@ rangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) PyObject *range; /* create a range object for pickling */ - start = PyLong_FromLong(r->start); + long lstart; + (void)_PyRangeIter_GetLengthAndStart(r, &lstart); + start = PyLong_FromLong(lstart); if (start == NULL) goto err; - stop = PyLong_FromLong(r->start + r->len * r->step); + stop = PyLong_FromLong(r->stop); if (stop == NULL) goto err; step = PyLong_FromLong(r->step); if (step == NULL) goto err; range = (PyObject*)make_range_object(&PyRange_Type, - start, stop, step); + start, stop, step); if (range == NULL) goto err; /* return the result */ @@ -889,12 +895,13 @@ rangeiter_setstate(PyObject *op, PyObject *state) if (index == -1 && PyErr_Occurred()) return NULL; /* silently clip the index value */ + long start; + long len = _PyRangeIter_GetLengthAndStart(r, &start); if (index < 0) index = 0; - else if (index > r->len) - index = r->len; /* exhausted iterator */ - r->start += index * r->step; - r->len -= index; + else if (index > len) + index = len; /* exhausted iterator */ + _PyRangeIter_SetLength(r, len - index); Py_RETURN_NONE; } @@ -989,9 +996,10 @@ fast_range_iter(long start, long stop, long step, long len) } } assert(Py_IS_TYPE(it, &PyRangeIter_Type)); - it->start = start; - it->step = step; it->len = len; + it->end = start + step * len; + it->step = step; + it->stop = stop; return (PyObject *)it; } @@ -1006,13 +1014,17 @@ static PyObject * longrangeiter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { longrangeiterobject *r = (longrangeiterobject*)op; - Py_INCREF(r->len); - return r->len; + PyObject *len; + Py_BEGIN_CRITICAL_SECTION(r); + len = Py_NewRef(r->len); + Py_END_CRITICAL_SECTION(); + return len; } static PyObject * -longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +longrangeiter_reduce_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); longrangeiterobject *r = (longrangeiterobject*)op; PyObject *product, *stop=NULL; PyObject *range; @@ -1025,8 +1037,8 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) Py_DECREF(product); if (stop == NULL) return NULL; - range = (PyObject*)make_range_object(&PyRange_Type, - Py_NewRef(r->start), stop, Py_NewRef(r->step)); + range = (PyObject*)make_range_object(&PyRange_Type, + Py_NewRef(r->start), stop, Py_NewRef(r->step)); if (range == NULL) { Py_DECREF(r->start); Py_DECREF(stop); @@ -1040,8 +1052,19 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) } static PyObject * -longrangeiter_setstate(PyObject *op, PyObject *state) +longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = longrangeiter_reduce_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + +static PyObject * +longrangeiter_setstate_lock_held(PyObject *op, PyObject *state) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; @@ -1079,6 +1102,16 @@ longrangeiter_setstate(PyObject *op, PyObject *state) Py_RETURN_NONE; } +static PyObject * +longrangeiter_setstate(PyObject *op, PyObject *state) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = longrangeiter_setstate_lock_held(op, state); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyMethodDef longrangeiter_methods[] = { {"__length_hint__", longrangeiter_len, METH_NOARGS, length_hint_doc}, {"__reduce__", longrangeiter_reduce, METH_NOARGS, reduce_doc}, @@ -1097,8 +1130,9 @@ longrangeiter_dealloc(PyObject *op) } static PyObject * -longrangeiter_next(PyObject *op) +longrangeiter_next_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); longrangeiterobject *r = (longrangeiterobject*)op; if (PyObject_RichCompareBool(r->len, _PyLong_GetZero(), Py_GT) != 1) return NULL; @@ -1118,6 +1152,16 @@ longrangeiter_next(PyObject *op) return result; } +static PyObject * +longrangeiter_next(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = longrangeiter_next_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + PyTypeObject PyLongRangeIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "longrange_iterator", /* tp_name */ @@ -1310,3 +1354,14 @@ range_reverse(PyObject *seq, PyObject *Py_UNUSED(ignored)) Py_DECREF(it); return NULL; } + +long +_PyRangeIter_GetLength(_PyRangeIterObject *r, long start) +{ +#ifdef Py_GIL_DISABLED + // we know this won't be greater than a long + return (long)get_len_of_range(start, r->stop, r->step); +#else + return r->len; +#endif +} diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8cedee31e08a00..c64904360aca63 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -13,10 +13,10 @@ unsigned char M_test_frozenmain[] = { 80,5,91,6,11,0,80,6,91,5,91,6,43,26,0,0, 0,0,0,0,0,0,0,0,11,0,48,4,50,1,0,0, 0,0,0,0,30,0,73,26,0,0,8,0,29,0,80,1, - 34,0,41,8,233,0,0,0,0,78,122,18,70,114,111,122, - 101,110,32,72,101,108,108,111,32,87,111,114,108,100,122,8, + 34,0,41,8,233,0,0,0,0,78,218,18,70,114,111,122, + 101,110,32,72,101,108,108,111,32,87,111,114,108,100,218,8, 115,121,115,46,97,114,103,118,218,6,99,111,110,102,105,103, - 122,7,99,111,110,102,105,103,32,122,2,58,32,41,5,218, + 218,7,99,111,110,102,105,103,32,218,2,58,32,41,5,218, 12,112,114,111,103,114,97,109,95,110,97,109,101,218,10,101, 120,101,99,117,116,97,98,108,101,218,15,117,115,101,95,101, 110,118,105,114,111,110,109,101,110,116,218,17,99,111,110,102, @@ -25,15 +25,15 @@ unsigned char M_test_frozenmain[] = { 3,115,121,115,218,17,95,116,101,115,116,105,110,116,101,114, 110,97,108,99,97,112,105,218,5,112,114,105,110,116,218,4, 97,114,103,118,218,11,103,101,116,95,99,111,110,102,105,103, - 115,114,3,0,0,0,218,3,107,101,121,169,0,243,0,0, + 115,114,5,0,0,0,218,3,107,101,121,169,0,243,0,0, 0,0,218,18,116,101,115,116,95,102,114,111,122,101,110,109, 97,105,110,46,112,121,218,8,60,109,111,100,117,108,101,62, - 114,18,0,0,0,1,0,0,0,115,94,0,0,0,240,3, + 114,22,0,0,0,1,0,0,0,115,94,0,0,0,240,3, 1,1,1,243,8,0,1,11,219,0,24,225,0,5,208,6, 26,212,0,27,217,0,5,128,106,144,35,151,40,145,40,212, 0,27,216,9,26,215,9,38,210,9,38,211,9,40,168,24, 213,9,50,128,6,243,2,6,12,2,128,67,241,14,0,5, 10,136,71,144,67,144,53,152,2,152,54,160,35,157,59,152, - 45,208,10,40,214,4,41,243,15,6,12,2,114,16,0,0, + 45,208,10,40,214,4,41,243,15,6,12,2,114,20,0,0, 0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 53da324ee5ab89..afc79c1417a0f2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3290,42 +3290,33 @@ dummy_func( op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); -#ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)r)); -#endif } - replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { + replaced op(_ITER_NEXT_RANGE, (iter -- iter, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); -#endif STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { + long value; + long len = _PyRangeIter_GetLengthAndStart(r, &value); + if (len <= 0) { // Jump over END_FOR instruction. JUMPBY(oparg + 1); DISPATCH(); } + _PyRangeIter_SetLength(r, len - 1); + PyObject *res = PyLong_FromLong(value); + ERROR_IF(res == NULL, error); + next = PyStackRef_FromPyObjectSteal(res); } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { + op(_ITER_NEXT_RANGE_TIER_TWO, (iter -- iter, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - EXIT_IF(r->len <= 0); - } - - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); -#endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; + long value; + long len = _PyRangeIter_GetLengthAndStart(r, &value); + EXIT_IF(len <= 0); + _PyRangeIter_SetLength(r, len - 1); PyObject *res = PyLong_FromLong(value); ERROR_IF(res == NULL, error); next = PyStackRef_FromPyObjectSteal(res); @@ -3334,7 +3325,6 @@ dummy_func( macro(FOR_ITER_RANGE) = unused/1 + // Skip over the counter _ITER_CHECK_RANGE + - _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3457ff0d6a1c06..3d7bd01fc24e55 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4291,42 +4291,24 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #endif break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ + /* _ITER_NEXT_RANGE is not a viable micro-op for tier 2 because it is replaced */ - case _GUARD_NOT_EXHAUSTED_RANGE: { + case _ITER_NEXT_RANGE_TIER_TWO: { _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) { + long value; + long len = _PyRangeIter_GetLengthAndStart(r, &value); + if (len <= 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - break; - } - - case _ITER_NEXT_RANGE: { - _PyStackRef iter; - _PyStackRef next; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; + _PyRangeIter_SetLength(r, len - 1); PyObject *res = PyLong_FromLong(value); if (res == NULL) { JUMP_TO_ERROR(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fb4ab92c635d9e..bc38d82bb8124b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5664,38 +5664,19 @@ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #endif } - // _ITER_JUMP_RANGE + // _ITER_NEXT_RANGE { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { + long value; + long len = _PyRangeIter_GetLengthAndStart(r, &value); + if (len <= 0) { JUMPBY(oparg + 1); DISPATCH(); } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; + _PyRangeIter_SetLength(r, len - 1); PyObject *res = PyLong_FromLong(value); if (res == NULL) { JUMP_TO_LABEL(error); diff --git a/Python/getargs.c b/Python/getargs.c index 022a4115038902..08325ca5a87c49 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -466,6 +466,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; + int istuple = PyTuple_Check(arg); + int mustbetuple = istuple; for (;;) { int c = *format++; @@ -481,51 +483,104 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (c == ':' || c == ';' || c == '\0') break; - else if (level == 0 && Py_ISALPHA(c) && c != 'e') - n++; + else { + if (level == 0 && Py_ISALPHA(c)) { + n++; + } + if (c == 'e' && (*format == 's' || *format == 't')) { + format++; + continue; + } + if (!mustbetuple) { + switch (c) { + case 'y': + case 's': + case 'z': + if (*format != '*') { + mustbetuple = 1; + } + break; + case 'S': + case 'Y': + case 'U': + mustbetuple = 1; + break; + case 'O': + if (*format != '&') { + mustbetuple = 1; + } + break; + } + } + } } - if (!PySequence_Check(arg) || PyBytes_Check(arg)) { + if (istuple) { + /* fallthrough */ + } + else if (!PySequence_Check(arg) || + PyUnicode_Check(arg) || PyBytes_Check(arg) || PyByteArray_Check(arg)) + { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item sequence, not %.50s", + "must be %d-item tuple, not %.50s", n, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } + else { + if (mustbetuple) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 0, + "argument must be %d-item tuple, not %T", n, arg)) + { + return msgbuf; + } + } + len = PySequence_Size(arg); + if (len != n) { + levels[0] = 0; + PyOS_snprintf(msgbuf, bufsize, + "must be %s of length %d, not %zd", + mustbetuple ? "tuple" : "sequence", n, len); + return msgbuf; + } + arg = PySequence_Tuple(arg); + if (arg == NULL) { + return msgbuf; + } + } - len = PySequence_Size(arg); + len = PyTuple_GET_SIZE(arg); if (len != n) { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be sequence of length %d, not %zd", + "must be tuple of length %d, not %zd", n, len); + if (!istuple) { + Py_DECREF(arg); + } return msgbuf; } format = *p_format; for (i = 0; i < n; i++) { const char *msg; - PyObject *item; - item = PySequence_GetItem(arg, i); - if (item == NULL) { - PyErr_Clear(); - levels[0] = i+1; - levels[1] = 0; - strncpy(msgbuf, "is not retrievable", bufsize); - return msgbuf; - } + PyObject *item = PyTuple_GET_ITEM(arg, i); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); - /* PySequence_GetItem calls tp->sq_item, which INCREFs */ - Py_XDECREF(item); if (msg != NULL) { levels[0] = i+1; + if (!istuple) { + Py_DECREF(arg); + } return msg; } } *p_format = format; + if (!istuple) { + Py_DECREF(arg); + } return NULL; } diff --git a/Python/jit.c b/Python/jit.c index 8a91d2f62a4627..7556e518feca09 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -19,6 +19,7 @@ #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" #include "pycore_pyerrors.h" +#include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" #include "pycore_tuple.h" diff --git a/Python/optimizer.c b/Python/optimizer.c index f8d0aa04b9e003..d9a2a0602935fc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -369,16 +369,16 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t _PyUOp_Replacements[MAX_UOP_ID + 1] = { - [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_NEXT_RANGE] = _ITER_NEXT_RANGE_TIER_TWO, }; static const uint8_t is_for_iter_test[MAX_UOP_ID + 1] = { - [_GUARD_NOT_EXHAUSTED_RANGE] = 1, + [_ITER_NEXT_RANGE_TIER_TWO] = 1, [_GUARD_NOT_EXHAUSTED_LIST] = 1, [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, [_FOR_ITER_TIER_TWO] = 1, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 870c32d74ac913..bf3dbeb4c77fd9 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1462,15 +1462,11 @@ break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + /* _ITER_NEXT_RANGE is not a viable micro-op for tier 2 */ - case _GUARD_NOT_EXHAUSTED_RANGE: { - break; - } - - case _ITER_NEXT_RANGE: { + case _ITER_NEXT_RANGE_TIER_TWO: { JitOptSymbol *next; - next = sym_new_type(ctx, &PyLong_Type); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a217d7136a5401..9c0f0ab3790e18 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -678,6 +678,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "JUMP_TO_LABEL", "restart_backoff_counter", "_Py_ReachedRecursionLimit", + "_PyRangeIter_GetLengthAndStart", + "_PyRangeIter_SetLength", ) def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: