From 94664176280c6cf50c5014fa31f1c72021076930 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 21:28:51 +0800 Subject: [PATCH 01/24] Skip refcounting where possible for common float ops --- Include/internal/pycore_optimizer.h | 41 ++- Include/internal/pycore_uop_ids.h | 461 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 12 + Lib/test/test_capi/test_opt.py | 20 +- Python/bytecodes.c | 46 +++ Python/executor_cases.c.h | 81 +++++ Python/optimizer_analysis.c | 10 + Python/optimizer_bytecodes.c | 16 + Python/optimizer_cases.c.h | 40 +++ Python/optimizer_symbols.c | 35 +- 10 files changed, 523 insertions(+), 239 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d3674726997f6a..e8b61096f477a1 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,38 +180,63 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; +#define DONT_SKIP_REFCOUNT 0 +#define SKIP_REFCOUNT 1 + typedef struct _jit_opt_known_class { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint32_t version; PyTypeObject *type; } JitOptKnownClass; typedef struct _jit_opt_known_version { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint32_t version; } JitOptKnownVersion; typedef struct _jit_opt_known_value { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; PyObject *value; } JitOptKnownValue; #define MAX_SYMBOLIC_TUPLE_SIZE 7 typedef struct _jit_opt_tuple { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; } JitOptTuple; typedef struct { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; bool invert; uint16_t value; } JitOptTruthiness; typedef union _jit_opt_symbol { - uint8_t tag; + struct { + uint8_t tag; + // Whether this object skips refcount on the stack + // (using the _PyStackRef API), or not. + // 0 - normal refcounting + // 1 - skip refcounting + int8_t skip_refcount; + }; JitOptKnownClass cls; JitOptKnownValue value; JitOptKnownVersion version; @@ -283,6 +308,10 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); +extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); + extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d08799487fda42..e48da341b63f1a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,22 +13,25 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT 311 -#define _BINARY_OP_SUBSCR_INIT_CALL 312 -#define _BINARY_OP_SUBSCR_LIST_INT 313 -#define _BINARY_OP_SUBSCR_LIST_SLICE 314 -#define _BINARY_OP_SUBSCR_STR_INT 315 -#define _BINARY_OP_SUBSCR_TUPLE_INT 316 -#define _BINARY_OP_SUBTRACT_FLOAT 317 -#define _BINARY_OP_SUBTRACT_INT 318 -#define _BINARY_SLICE 319 +#define _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_ADD_UNICODE 306 +#define _BINARY_OP_EXTEND 307 +#define _BINARY_OP_INPLACE_ADD_UNICODE 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF 310 +#define _BINARY_OP_MULTIPLY_INT 311 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 +#define _BINARY_OP_SUBSCR_DICT 313 +#define _BINARY_OP_SUBSCR_INIT_CALL 314 +#define _BINARY_OP_SUBSCR_LIST_INT 315 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -37,135 +40,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 320 -#define _CALL_BUILTIN_FAST 321 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 322 -#define _CALL_BUILTIN_O 323 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 324 -#define _CALL_KW_NON_PY 325 -#define _CALL_LEN 326 -#define _CALL_LIST_APPEND 327 -#define _CALL_METHOD_DESCRIPTOR_FAST 328 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 329 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 330 -#define _CALL_METHOD_DESCRIPTOR_O 331 -#define _CALL_NON_PY_GENERAL 332 -#define _CALL_STR_1 333 -#define _CALL_TUPLE_1 334 -#define _CALL_TYPE_1 335 -#define _CHECK_AND_ALLOCATE_OBJECT 336 -#define _CHECK_ATTR_CLASS 337 -#define _CHECK_ATTR_METHOD_LAZY_DICT 338 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 339 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 340 -#define _CHECK_FUNCTION_EXACT_ARGS 341 -#define _CHECK_FUNCTION_VERSION 342 -#define _CHECK_FUNCTION_VERSION_INLINE 343 -#define _CHECK_FUNCTION_VERSION_KW 344 -#define _CHECK_IS_NOT_PY_CALLABLE 345 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 346 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 347 -#define _CHECK_METHOD_VERSION 348 -#define _CHECK_METHOD_VERSION_KW 349 -#define _CHECK_PEP_523 350 -#define _CHECK_PERIODIC 351 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 352 -#define _CHECK_RECURSION_REMAINING 353 -#define _CHECK_STACK_SPACE 354 -#define _CHECK_STACK_SPACE_OPERAND 355 -#define _CHECK_VALIDITY 356 -#define _COMPARE_OP 357 -#define _COMPARE_OP_FLOAT 358 -#define _COMPARE_OP_INT 359 -#define _COMPARE_OP_STR 360 -#define _CONTAINS_OP 361 -#define _CONTAINS_OP_DICT 362 -#define _CONTAINS_OP_SET 363 +#define _CHECK_FUNCTION 343 +#define _CHECK_FUNCTION_EXACT_ARGS 344 +#define _CHECK_FUNCTION_VERSION 345 +#define _CHECK_FUNCTION_VERSION_INLINE 346 +#define _CHECK_FUNCTION_VERSION_KW 347 +#define _CHECK_IS_NOT_PY_CALLABLE 348 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 +#define _CHECK_METHOD_VERSION 351 +#define _CHECK_METHOD_VERSION_KW 352 +#define _CHECK_PEP_523 353 +#define _CHECK_PERIODIC 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COMPARE_OP 360 +#define _COMPARE_OP_FLOAT 361 +#define _COMPARE_OP_INT 362 +#define _COMPARE_OP_STR 363 +#define _CONTAINS_OP 364 +#define _CONTAINS_OP_DICT 365 +#define _CONTAINS_OP_SET 366 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 364 +#define _CREATE_INIT_FRAME 367 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 365 +#define _DEOPT 368 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 366 -#define _DO_CALL_FUNCTION_EX 367 -#define _DO_CALL_KW 368 +#define _DO_CALL 369 +#define _DO_CALL_FUNCTION_EX 370 +#define _DO_CALL_KW 371 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 369 +#define _ERROR_POP_N 372 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 370 -#define _EXPAND_METHOD_KW 371 -#define _FATAL_ERROR 372 +#define _EXPAND_METHOD 373 +#define _EXPAND_METHOD_KW 374 +#define _FATAL_ERROR 375 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 373 -#define _FOR_ITER_GEN_FRAME 374 -#define _FOR_ITER_TIER_TWO 375 +#define _FOR_ITER 376 +#define _FOR_ITER_GEN_FRAME 377 +#define _FOR_ITER_TIER_TWO 378 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 376 -#define _GUARD_CALLABLE_ISINSTANCE 377 -#define _GUARD_CALLABLE_LEN 378 -#define _GUARD_CALLABLE_LIST_APPEND 379 -#define _GUARD_CALLABLE_STR_1 380 -#define _GUARD_CALLABLE_TUPLE_1 381 -#define _GUARD_CALLABLE_TYPE_1 382 -#define _GUARD_DORV_NO_DICT 383 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 384 -#define _GUARD_GLOBALS_VERSION 385 -#define _GUARD_IS_FALSE_POP 386 -#define _GUARD_IS_NONE_POP 387 -#define _GUARD_IS_NOT_NONE_POP 388 -#define _GUARD_IS_TRUE_POP 389 -#define _GUARD_KEYS_VERSION 390 -#define _GUARD_NOS_DICT 391 -#define _GUARD_NOS_FLOAT 392 -#define _GUARD_NOS_INT 393 -#define _GUARD_NOS_LIST 394 -#define _GUARD_NOS_NOT_NULL 395 -#define _GUARD_NOS_NULL 396 -#define _GUARD_NOS_TUPLE 397 -#define _GUARD_NOS_UNICODE 398 -#define _GUARD_NOT_EXHAUSTED_LIST 399 -#define _GUARD_NOT_EXHAUSTED_RANGE 400 -#define _GUARD_NOT_EXHAUSTED_TUPLE 401 -#define _GUARD_THIRD_NULL 402 -#define _GUARD_TOS_ANY_SET 403 -#define _GUARD_TOS_DICT 404 -#define _GUARD_TOS_FLOAT 405 -#define _GUARD_TOS_INT 406 -#define _GUARD_TOS_LIST 407 -#define _GUARD_TOS_SLICE 408 -#define _GUARD_TOS_TUPLE 409 -#define _GUARD_TOS_UNICODE 410 -#define _GUARD_TYPE_VERSION 411 -#define _GUARD_TYPE_VERSION_AND_LOCK 412 +#define _GUARD_BINARY_OP_EXTEND 379 +#define _GUARD_CALLABLE_ISINSTANCE 380 +#define _GUARD_CALLABLE_LEN 381 +#define _GUARD_CALLABLE_LIST_APPEND 382 +#define _GUARD_CALLABLE_STR_1 383 +#define _GUARD_CALLABLE_TUPLE_1 384 +#define _GUARD_CALLABLE_TYPE_1 385 +#define _GUARD_DORV_NO_DICT 386 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 +#define _GUARD_GLOBALS_VERSION 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _GUARD_IS_TRUE_POP 392 +#define _GUARD_KEYS_VERSION 393 +#define _GUARD_NOS_DICT 394 +#define _GUARD_NOS_FLOAT 395 +#define _GUARD_NOS_INT 396 +#define _GUARD_NOS_LIST 397 +#define _GUARD_NOS_NOT_NULL 398 +#define _GUARD_NOS_NULL 399 +#define _GUARD_NOS_TUPLE 400 +#define _GUARD_NOS_UNICODE 401 +#define _GUARD_NOT_EXHAUSTED_LIST 402 +#define _GUARD_NOT_EXHAUSTED_RANGE 403 +#define _GUARD_NOT_EXHAUSTED_TUPLE 404 +#define _GUARD_THIRD_NULL 405 +#define _GUARD_TOS_ANY_SET 406 +#define _GUARD_TOS_DICT 407 +#define _GUARD_TOS_FLOAT 408 +#define _GUARD_TOS_INT 409 +#define _GUARD_TOS_LIST 410 +#define _GUARD_TOS_SLICE 411 +#define _GUARD_TOS_TUPLE 412 +#define _GUARD_TOS_UNICODE 413 +#define _GUARD_TYPE_VERSION 414 +#define _GUARD_TYPE_VERSION_AND_LOCK 415 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 413 -#define _INIT_CALL_PY_EXACT_ARGS 414 -#define _INIT_CALL_PY_EXACT_ARGS_0 415 -#define _INIT_CALL_PY_EXACT_ARGS_1 416 -#define _INIT_CALL_PY_EXACT_ARGS_2 417 -#define _INIT_CALL_PY_EXACT_ARGS_3 418 -#define _INIT_CALL_PY_EXACT_ARGS_4 419 -#define _INSERT_NULL 420 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 +#define _INIT_CALL_PY_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS_0 418 +#define _INIT_CALL_PY_EXACT_ARGS_1 419 +#define _INIT_CALL_PY_EXACT_ARGS_2 420 +#define _INIT_CALL_PY_EXACT_ARGS_3 421 +#define _INIT_CALL_PY_EXACT_ARGS_4 422 +#define _INSERT_NULL 423 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -175,170 +178,170 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 421 +#define _IS_NONE 424 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 422 -#define _ITER_CHECK_RANGE 423 -#define _ITER_CHECK_TUPLE 424 -#define _ITER_JUMP_LIST 425 -#define _ITER_JUMP_RANGE 426 -#define _ITER_JUMP_TUPLE 427 -#define _ITER_NEXT_LIST 428 -#define _ITER_NEXT_LIST_TIER_TWO 429 -#define _ITER_NEXT_RANGE 430 -#define _ITER_NEXT_TUPLE 431 -#define _JUMP_TO_TOP 432 +#define _ITER_CHECK_LIST 425 +#define _ITER_CHECK_RANGE 426 +#define _ITER_CHECK_TUPLE 427 +#define _ITER_JUMP_LIST 428 +#define _ITER_JUMP_RANGE 429 +#define _ITER_JUMP_TUPLE 430 +#define _ITER_NEXT_LIST 431 +#define _ITER_NEXT_LIST_TIER_TWO 432 +#define _ITER_NEXT_RANGE 433 +#define _ITER_NEXT_TUPLE 434 +#define _JUMP_TO_TOP 435 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 433 -#define _LOAD_ATTR_CLASS 434 +#define _LOAD_ATTR 436 +#define _LOAD_ATTR_CLASS 437 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 435 -#define _LOAD_ATTR_METHOD_LAZY_DICT 436 -#define _LOAD_ATTR_METHOD_NO_DICT 437 -#define _LOAD_ATTR_METHOD_WITH_VALUES 438 -#define _LOAD_ATTR_MODULE 439 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 440 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 441 -#define _LOAD_ATTR_PROPERTY_FRAME 442 -#define _LOAD_ATTR_SLOT 443 -#define _LOAD_ATTR_WITH_HINT 444 +#define _LOAD_ATTR_INSTANCE_VALUE 438 +#define _LOAD_ATTR_METHOD_LAZY_DICT 439 +#define _LOAD_ATTR_METHOD_NO_DICT 440 +#define _LOAD_ATTR_METHOD_WITH_VALUES 441 +#define _LOAD_ATTR_MODULE 442 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 +#define _LOAD_ATTR_PROPERTY_FRAME 445 +#define _LOAD_ATTR_SLOT 446 +#define _LOAD_ATTR_WITH_HINT 447 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 445 +#define _LOAD_BYTECODE 448 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 446 -#define _LOAD_CONST_INLINE_BORROW 447 -#define _LOAD_CONST_UNDER_INLINE 448 -#define _LOAD_CONST_UNDER_INLINE_BORROW 449 +#define _LOAD_CONST_INLINE 449 +#define _LOAD_CONST_INLINE_BORROW 450 +#define _LOAD_CONST_UNDER_INLINE 451 +#define _LOAD_CONST_UNDER_INLINE_BORROW 452 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 450 -#define _LOAD_FAST_0 451 -#define _LOAD_FAST_1 452 -#define _LOAD_FAST_2 453 -#define _LOAD_FAST_3 454 -#define _LOAD_FAST_4 455 -#define _LOAD_FAST_5 456 -#define _LOAD_FAST_6 457 -#define _LOAD_FAST_7 458 +#define _LOAD_FAST 453 +#define _LOAD_FAST_0 454 +#define _LOAD_FAST_1 455 +#define _LOAD_FAST_2 456 +#define _LOAD_FAST_3 457 +#define _LOAD_FAST_4 458 +#define _LOAD_FAST_5 459 +#define _LOAD_FAST_6 460 +#define _LOAD_FAST_7 461 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 459 -#define _LOAD_FAST_BORROW_0 460 -#define _LOAD_FAST_BORROW_1 461 -#define _LOAD_FAST_BORROW_2 462 -#define _LOAD_FAST_BORROW_3 463 -#define _LOAD_FAST_BORROW_4 464 -#define _LOAD_FAST_BORROW_5 465 -#define _LOAD_FAST_BORROW_6 466 -#define _LOAD_FAST_BORROW_7 467 +#define _LOAD_FAST_BORROW 462 +#define _LOAD_FAST_BORROW_0 463 +#define _LOAD_FAST_BORROW_1 464 +#define _LOAD_FAST_BORROW_2 465 +#define _LOAD_FAST_BORROW_3 466 +#define _LOAD_FAST_BORROW_4 467 +#define _LOAD_FAST_BORROW_5 468 +#define _LOAD_FAST_BORROW_6 469 +#define _LOAD_FAST_BORROW_7 470 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 468 -#define _LOAD_GLOBAL_BUILTINS 469 -#define _LOAD_GLOBAL_MODULE 470 +#define _LOAD_GLOBAL 471 +#define _LOAD_GLOBAL_BUILTINS 472 +#define _LOAD_GLOBAL_MODULE 473 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 471 -#define _LOAD_SMALL_INT_0 472 -#define _LOAD_SMALL_INT_1 473 -#define _LOAD_SMALL_INT_2 474 -#define _LOAD_SMALL_INT_3 475 -#define _LOAD_SPECIAL 476 +#define _LOAD_SMALL_INT 474 +#define _LOAD_SMALL_INT_0 475 +#define _LOAD_SMALL_INT_1 476 +#define _LOAD_SMALL_INT_2 477 +#define _LOAD_SMALL_INT_3 478 +#define _LOAD_SPECIAL 479 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 477 +#define _MAKE_CALLARGS_A_TUPLE 480 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 478 +#define _MAKE_WARM 481 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 479 -#define _MAYBE_EXPAND_METHOD_KW 480 -#define _MONITOR_CALL 481 -#define _MONITOR_CALL_KW 482 -#define _MONITOR_JUMP_BACKWARD 483 -#define _MONITOR_RESUME 484 +#define _MAYBE_EXPAND_METHOD 482 +#define _MAYBE_EXPAND_METHOD_KW 483 +#define _MONITOR_CALL 484 +#define _MONITOR_CALL_KW 485 +#define _MONITOR_JUMP_BACKWARD 486 +#define _MONITOR_RESUME 487 #define _NOP NOP -#define _POP_CALL 485 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 486 -#define _POP_CALL_ONE 487 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 488 -#define _POP_CALL_TWO 489 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 490 +#define _POP_CALL 488 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 +#define _POP_CALL_ONE 490 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 +#define _POP_CALL_TWO 492 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 491 -#define _POP_JUMP_IF_TRUE 492 +#define _POP_JUMP_IF_FALSE 494 +#define _POP_JUMP_IF_TRUE 495 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 493 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 494 -#define _POP_TWO 495 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 496 +#define _POP_TOP_LOAD_CONST_INLINE 496 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 +#define _POP_TWO 498 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 497 +#define _PUSH_FRAME 500 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 498 -#define _PY_FRAME_GENERAL 499 -#define _PY_FRAME_KW 500 -#define _QUICKEN_RESUME 501 -#define _REPLACE_WITH_TRUE 502 +#define _PUSH_NULL_CONDITIONAL 501 +#define _PY_FRAME_GENERAL 502 +#define _PY_FRAME_KW 503 +#define _QUICKEN_RESUME 504 +#define _REPLACE_WITH_TRUE 505 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 503 -#define _SEND 504 -#define _SEND_GEN_FRAME 505 +#define _SAVE_RETURN_OFFSET 506 +#define _SEND 507 +#define _SEND_GEN_FRAME 508 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 506 -#define _STORE_ATTR 507 -#define _STORE_ATTR_INSTANCE_VALUE 508 -#define _STORE_ATTR_SLOT 509 -#define _STORE_ATTR_WITH_HINT 510 +#define _START_EXECUTOR 509 +#define _STORE_ATTR 510 +#define _STORE_ATTR_INSTANCE_VALUE 511 +#define _STORE_ATTR_SLOT 512 +#define _STORE_ATTR_WITH_HINT 513 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 511 -#define _STORE_FAST_0 512 -#define _STORE_FAST_1 513 -#define _STORE_FAST_2 514 -#define _STORE_FAST_3 515 -#define _STORE_FAST_4 516 -#define _STORE_FAST_5 517 -#define _STORE_FAST_6 518 -#define _STORE_FAST_7 519 +#define _STORE_FAST 514 +#define _STORE_FAST_0 515 +#define _STORE_FAST_1 516 +#define _STORE_FAST_2 517 +#define _STORE_FAST_3 518 +#define _STORE_FAST_4 519 +#define _STORE_FAST_5 520 +#define _STORE_FAST_6 521 +#define _STORE_FAST_7 522 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 520 -#define _STORE_SUBSCR 521 -#define _STORE_SUBSCR_DICT 522 -#define _STORE_SUBSCR_LIST_INT 523 +#define _STORE_SLICE 523 +#define _STORE_SUBSCR 524 +#define _STORE_SUBSCR_DICT 525 +#define _STORE_SUBSCR_LIST_INT 526 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 524 -#define _TO_BOOL 525 +#define _TIER2_RESUME_CHECK 527 +#define _TO_BOOL 528 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 526 +#define _TO_BOOL_LIST 529 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 527 +#define _TO_BOOL_STR 530 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 528 -#define _UNPACK_SEQUENCE_LIST 529 -#define _UNPACK_SEQUENCE_TUPLE 530 -#define _UNPACK_SEQUENCE_TWO_TUPLE 531 +#define _UNPACK_SEQUENCE 531 +#define _UNPACK_SEQUENCE_LIST 532 +#define _UNPACK_SEQUENCE_TUPLE 533 +#define _UNPACK_SEQUENCE_TWO_TUPLE 534 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 531 +#define MAX_UOP_ID 534 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5ebe124983b862..978bc523999b36 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,6 +92,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -333,11 +336,13 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", @@ -347,6 +352,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -782,6 +788,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: + return 2; + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index cb6eae484149ee..bb8090393949a7 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -678,7 +678,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -2219,6 +2219,20 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_float_op_refcount_elimination(self): + def testfunc(args): + a, b, n = args + c = 0.0 + for _ in range(n): + c += a + b + return c + + res, ex = self._run_with_optimizer(testfunc, (0.1, 0.1, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be22c7446f5402..0d9684da51b35c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -677,6 +677,52 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } + + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 40090e692e4a72..49e0cb2ebaba62 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1043,6 +1043,87 @@ break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { _PyStackRef right; _PyStackRef left; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 851e1efa0497af..02022382cb4a3a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,6 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness +#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount +#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount +#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount static int optimize_to_bool( @@ -440,6 +443,13 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +// TODO (gh-134584) generate most of this table automatically +const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, +}; + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 49c6bfb6c1b01a..aa90d6b7645259 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,20 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); + sym_set_skip_refcount(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -296,6 +300,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { @@ -316,6 +324,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { @@ -336,6 +348,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bf7ac72d4579e7..75bfcaf30b994a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,6 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -40,6 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -49,6 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); + sym_set_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -60,6 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -434,6 +438,9 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } @@ -463,6 +470,9 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } @@ -492,10 +502,40 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { JitOptSymbol *right; JitOptSymbol *left; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 25de5d83166f64..68f1ecd1d316a4 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,6 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; + self->skip_refcount = DONT_SKIP_REFCOUNT; return self; } @@ -106,6 +107,9 @@ static void make_const(JitOptSymbol *sym, PyObject *val) { sym->tag = JIT_SYM_KNOWN_VALUE_TAG; sym->value.value = Py_NewRef(val); + // Constants don't need to be refcounted, as they are always + // kept alive by co_consts. + sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -381,6 +385,25 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } +void +_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + sym->skip_refcount = SKIP_REFCOUNT; +} + +void +_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + sym->skip_refcount = DONT_SKIP_REFCOUNT; +} + +bool +_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); + return sym->skip_refcount == SKIP_REFCOUNT; +} + JitOptSymbol * _Py_uop_sym_new_unknown(JitOptContext *ctx) @@ -709,7 +732,7 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); + static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. @@ -771,6 +794,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -819,6 +843,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -847,8 +872,16 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_skip_refcount(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From ad03b1eeda291ac7f0ca414c2703de714c46fcce Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:06:07 +0800 Subject: [PATCH 02/24] Add for common list ops --- Include/internal/pycore_uop_ids.h | 442 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 8 + Python/bytecodes.c | 54 +++ Python/executor_cases.c.h | 83 +++++ Python/optimizer_analysis.c | 2 + Python/optimizer_bytecodes.c | 17 + Python/optimizer_cases.c.h | 29 ++ 7 files changed, 415 insertions(+), 220 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index e48da341b63f1a..a8e8cce09a458e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -25,13 +25,14 @@ extern "C" { #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_SLICE 316 -#define _BINARY_OP_SUBSCR_STR_INT 317 -#define _BINARY_OP_SUBSCR_TUPLE_INT 318 -#define _BINARY_OP_SUBTRACT_FLOAT 319 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 320 -#define _BINARY_OP_SUBTRACT_INT 321 -#define _BINARY_SLICE 322 +#define _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF 316 +#define _BINARY_OP_SUBSCR_LIST_SLICE 317 +#define _BINARY_OP_SUBSCR_STR_INT 318 +#define _BINARY_OP_SUBSCR_TUPLE_INT 319 +#define _BINARY_OP_SUBTRACT_FLOAT 320 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 321 +#define _BINARY_OP_SUBTRACT_INT 322 +#define _BINARY_SLICE 323 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -40,135 +41,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 323 -#define _CALL_BUILTIN_FAST 324 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 -#define _CALL_BUILTIN_O 326 +#define _CALL_BUILTIN_CLASS 324 +#define _CALL_BUILTIN_FAST 325 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 326 +#define _CALL_BUILTIN_O 327 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_ISINSTANCE 328 +#define _CALL_KW_NON_PY 329 +#define _CALL_LEN 330 +#define _CALL_LIST_APPEND 331 +#define _CALL_METHOD_DESCRIPTOR_FAST 332 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 333 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 334 +#define _CALL_METHOD_DESCRIPTOR_O 335 +#define _CALL_NON_PY_GENERAL 336 +#define _CALL_STR_1 337 +#define _CALL_TUPLE_1 338 +#define _CALL_TYPE_1 339 +#define _CHECK_AND_ALLOCATE_OBJECT 340 +#define _CHECK_ATTR_CLASS 341 +#define _CHECK_ATTR_METHOD_LAZY_DICT 342 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 343 -#define _CHECK_FUNCTION_EXACT_ARGS 344 -#define _CHECK_FUNCTION_VERSION 345 -#define _CHECK_FUNCTION_VERSION_INLINE 346 -#define _CHECK_FUNCTION_VERSION_KW 347 -#define _CHECK_IS_NOT_PY_CALLABLE 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 -#define _CHECK_METHOD_VERSION 351 -#define _CHECK_METHOD_VERSION_KW 352 -#define _CHECK_PEP_523 353 -#define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COMPARE_OP 360 -#define _COMPARE_OP_FLOAT 361 -#define _COMPARE_OP_INT 362 -#define _COMPARE_OP_STR 363 -#define _CONTAINS_OP 364 -#define _CONTAINS_OP_DICT 365 -#define _CONTAINS_OP_SET 366 +#define _CHECK_FUNCTION 344 +#define _CHECK_FUNCTION_EXACT_ARGS 345 +#define _CHECK_FUNCTION_VERSION 346 +#define _CHECK_FUNCTION_VERSION_INLINE 347 +#define _CHECK_FUNCTION_VERSION_KW 348 +#define _CHECK_IS_NOT_PY_CALLABLE 349 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 350 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 351 +#define _CHECK_METHOD_VERSION 352 +#define _CHECK_METHOD_VERSION_KW 353 +#define _CHECK_PEP_523 354 +#define _CHECK_PERIODIC 355 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 +#define _CHECK_RECURSION_REMAINING 357 +#define _CHECK_STACK_SPACE 358 +#define _CHECK_STACK_SPACE_OPERAND 359 +#define _CHECK_VALIDITY 360 +#define _COMPARE_OP 361 +#define _COMPARE_OP_FLOAT 362 +#define _COMPARE_OP_INT 363 +#define _COMPARE_OP_STR 364 +#define _CONTAINS_OP 365 +#define _CONTAINS_OP_DICT 366 +#define _CONTAINS_OP_SET 367 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 367 +#define _CREATE_INIT_FRAME 368 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 368 +#define _DEOPT 369 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 369 -#define _DO_CALL_FUNCTION_EX 370 -#define _DO_CALL_KW 371 +#define _DO_CALL 370 +#define _DO_CALL_FUNCTION_EX 371 +#define _DO_CALL_KW 372 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 372 +#define _ERROR_POP_N 373 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 373 -#define _EXPAND_METHOD_KW 374 -#define _FATAL_ERROR 375 +#define _EXPAND_METHOD 374 +#define _EXPAND_METHOD_KW 375 +#define _FATAL_ERROR 376 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 376 -#define _FOR_ITER_GEN_FRAME 377 -#define _FOR_ITER_TIER_TWO 378 +#define _FOR_ITER 377 +#define _FOR_ITER_GEN_FRAME 378 +#define _FOR_ITER_TIER_TWO 379 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 379 -#define _GUARD_CALLABLE_ISINSTANCE 380 -#define _GUARD_CALLABLE_LEN 381 -#define _GUARD_CALLABLE_LIST_APPEND 382 -#define _GUARD_CALLABLE_STR_1 383 -#define _GUARD_CALLABLE_TUPLE_1 384 -#define _GUARD_CALLABLE_TYPE_1 385 -#define _GUARD_DORV_NO_DICT 386 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 -#define _GUARD_GLOBALS_VERSION 388 -#define _GUARD_IS_FALSE_POP 389 -#define _GUARD_IS_NONE_POP 390 -#define _GUARD_IS_NOT_NONE_POP 391 -#define _GUARD_IS_TRUE_POP 392 -#define _GUARD_KEYS_VERSION 393 -#define _GUARD_NOS_DICT 394 -#define _GUARD_NOS_FLOAT 395 -#define _GUARD_NOS_INT 396 -#define _GUARD_NOS_LIST 397 -#define _GUARD_NOS_NOT_NULL 398 -#define _GUARD_NOS_NULL 399 -#define _GUARD_NOS_TUPLE 400 -#define _GUARD_NOS_UNICODE 401 -#define _GUARD_NOT_EXHAUSTED_LIST 402 -#define _GUARD_NOT_EXHAUSTED_RANGE 403 -#define _GUARD_NOT_EXHAUSTED_TUPLE 404 -#define _GUARD_THIRD_NULL 405 -#define _GUARD_TOS_ANY_SET 406 -#define _GUARD_TOS_DICT 407 -#define _GUARD_TOS_FLOAT 408 -#define _GUARD_TOS_INT 409 -#define _GUARD_TOS_LIST 410 -#define _GUARD_TOS_SLICE 411 -#define _GUARD_TOS_TUPLE 412 -#define _GUARD_TOS_UNICODE 413 -#define _GUARD_TYPE_VERSION 414 -#define _GUARD_TYPE_VERSION_AND_LOCK 415 +#define _GUARD_BINARY_OP_EXTEND 380 +#define _GUARD_CALLABLE_ISINSTANCE 381 +#define _GUARD_CALLABLE_LEN 382 +#define _GUARD_CALLABLE_LIST_APPEND 383 +#define _GUARD_CALLABLE_STR_1 384 +#define _GUARD_CALLABLE_TUPLE_1 385 +#define _GUARD_CALLABLE_TYPE_1 386 +#define _GUARD_DORV_NO_DICT 387 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 388 +#define _GUARD_GLOBALS_VERSION 389 +#define _GUARD_IS_FALSE_POP 390 +#define _GUARD_IS_NONE_POP 391 +#define _GUARD_IS_NOT_NONE_POP 392 +#define _GUARD_IS_TRUE_POP 393 +#define _GUARD_KEYS_VERSION 394 +#define _GUARD_NOS_DICT 395 +#define _GUARD_NOS_FLOAT 396 +#define _GUARD_NOS_INT 397 +#define _GUARD_NOS_LIST 398 +#define _GUARD_NOS_NOT_NULL 399 +#define _GUARD_NOS_NULL 400 +#define _GUARD_NOS_TUPLE 401 +#define _GUARD_NOS_UNICODE 402 +#define _GUARD_NOT_EXHAUSTED_LIST 403 +#define _GUARD_NOT_EXHAUSTED_RANGE 404 +#define _GUARD_NOT_EXHAUSTED_TUPLE 405 +#define _GUARD_THIRD_NULL 406 +#define _GUARD_TOS_ANY_SET 407 +#define _GUARD_TOS_DICT 408 +#define _GUARD_TOS_FLOAT 409 +#define _GUARD_TOS_INT 410 +#define _GUARD_TOS_LIST 411 +#define _GUARD_TOS_SLICE 412 +#define _GUARD_TOS_TUPLE 413 +#define _GUARD_TOS_UNICODE 414 +#define _GUARD_TYPE_VERSION 415 +#define _GUARD_TYPE_VERSION_AND_LOCK 416 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 -#define _INIT_CALL_PY_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS_0 418 -#define _INIT_CALL_PY_EXACT_ARGS_1 419 -#define _INIT_CALL_PY_EXACT_ARGS_2 420 -#define _INIT_CALL_PY_EXACT_ARGS_3 421 -#define _INIT_CALL_PY_EXACT_ARGS_4 422 -#define _INSERT_NULL 423 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS 418 +#define _INIT_CALL_PY_EXACT_ARGS_0 419 +#define _INIT_CALL_PY_EXACT_ARGS_1 420 +#define _INIT_CALL_PY_EXACT_ARGS_2 421 +#define _INIT_CALL_PY_EXACT_ARGS_3 422 +#define _INIT_CALL_PY_EXACT_ARGS_4 423 +#define _INSERT_NULL 424 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -178,170 +179,171 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 424 +#define _IS_NONE 425 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 425 -#define _ITER_CHECK_RANGE 426 -#define _ITER_CHECK_TUPLE 427 -#define _ITER_JUMP_LIST 428 -#define _ITER_JUMP_RANGE 429 -#define _ITER_JUMP_TUPLE 430 -#define _ITER_NEXT_LIST 431 -#define _ITER_NEXT_LIST_TIER_TWO 432 -#define _ITER_NEXT_RANGE 433 -#define _ITER_NEXT_TUPLE 434 -#define _JUMP_TO_TOP 435 +#define _ITER_CHECK_LIST 426 +#define _ITER_CHECK_RANGE 427 +#define _ITER_CHECK_TUPLE 428 +#define _ITER_JUMP_LIST 429 +#define _ITER_JUMP_RANGE 430 +#define _ITER_JUMP_TUPLE 431 +#define _ITER_NEXT_LIST 432 +#define _ITER_NEXT_LIST_TIER_TWO 433 +#define _ITER_NEXT_RANGE 434 +#define _ITER_NEXT_TUPLE 435 +#define _JUMP_TO_TOP 436 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 436 -#define _LOAD_ATTR_CLASS 437 +#define _LOAD_ATTR 437 +#define _LOAD_ATTR_CLASS 438 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 438 -#define _LOAD_ATTR_METHOD_LAZY_DICT 439 -#define _LOAD_ATTR_METHOD_NO_DICT 440 -#define _LOAD_ATTR_METHOD_WITH_VALUES 441 -#define _LOAD_ATTR_MODULE 442 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 -#define _LOAD_ATTR_PROPERTY_FRAME 445 -#define _LOAD_ATTR_SLOT 446 -#define _LOAD_ATTR_WITH_HINT 447 +#define _LOAD_ATTR_INSTANCE_VALUE 439 +#define _LOAD_ATTR_METHOD_LAZY_DICT 440 +#define _LOAD_ATTR_METHOD_NO_DICT 441 +#define _LOAD_ATTR_METHOD_WITH_VALUES 442 +#define _LOAD_ATTR_MODULE 443 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 +#define _LOAD_ATTR_PROPERTY_FRAME 446 +#define _LOAD_ATTR_SLOT 447 +#define _LOAD_ATTR_WITH_HINT 448 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 448 +#define _LOAD_BYTECODE 449 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 449 -#define _LOAD_CONST_INLINE_BORROW 450 -#define _LOAD_CONST_UNDER_INLINE 451 -#define _LOAD_CONST_UNDER_INLINE_BORROW 452 +#define _LOAD_CONST_INLINE 450 +#define _LOAD_CONST_INLINE_BORROW 451 +#define _LOAD_CONST_UNDER_INLINE 452 +#define _LOAD_CONST_UNDER_INLINE_BORROW 453 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 453 -#define _LOAD_FAST_0 454 -#define _LOAD_FAST_1 455 -#define _LOAD_FAST_2 456 -#define _LOAD_FAST_3 457 -#define _LOAD_FAST_4 458 -#define _LOAD_FAST_5 459 -#define _LOAD_FAST_6 460 -#define _LOAD_FAST_7 461 +#define _LOAD_FAST 454 +#define _LOAD_FAST_0 455 +#define _LOAD_FAST_1 456 +#define _LOAD_FAST_2 457 +#define _LOAD_FAST_3 458 +#define _LOAD_FAST_4 459 +#define _LOAD_FAST_5 460 +#define _LOAD_FAST_6 461 +#define _LOAD_FAST_7 462 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 462 -#define _LOAD_FAST_BORROW_0 463 -#define _LOAD_FAST_BORROW_1 464 -#define _LOAD_FAST_BORROW_2 465 -#define _LOAD_FAST_BORROW_3 466 -#define _LOAD_FAST_BORROW_4 467 -#define _LOAD_FAST_BORROW_5 468 -#define _LOAD_FAST_BORROW_6 469 -#define _LOAD_FAST_BORROW_7 470 +#define _LOAD_FAST_BORROW 463 +#define _LOAD_FAST_BORROW_0 464 +#define _LOAD_FAST_BORROW_1 465 +#define _LOAD_FAST_BORROW_2 466 +#define _LOAD_FAST_BORROW_3 467 +#define _LOAD_FAST_BORROW_4 468 +#define _LOAD_FAST_BORROW_5 469 +#define _LOAD_FAST_BORROW_6 470 +#define _LOAD_FAST_BORROW_7 471 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 471 -#define _LOAD_GLOBAL_BUILTINS 472 -#define _LOAD_GLOBAL_MODULE 473 +#define _LOAD_GLOBAL 472 +#define _LOAD_GLOBAL_BUILTINS 473 +#define _LOAD_GLOBAL_MODULE 474 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 474 -#define _LOAD_SMALL_INT_0 475 -#define _LOAD_SMALL_INT_1 476 -#define _LOAD_SMALL_INT_2 477 -#define _LOAD_SMALL_INT_3 478 -#define _LOAD_SPECIAL 479 +#define _LOAD_SMALL_INT 475 +#define _LOAD_SMALL_INT_0 476 +#define _LOAD_SMALL_INT_1 477 +#define _LOAD_SMALL_INT_2 478 +#define _LOAD_SMALL_INT_3 479 +#define _LOAD_SPECIAL 480 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 480 +#define _MAKE_CALLARGS_A_TUPLE 481 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 481 +#define _MAKE_WARM 482 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 482 -#define _MAYBE_EXPAND_METHOD_KW 483 -#define _MONITOR_CALL 484 -#define _MONITOR_CALL_KW 485 -#define _MONITOR_JUMP_BACKWARD 486 -#define _MONITOR_RESUME 487 +#define _MAYBE_EXPAND_METHOD 483 +#define _MAYBE_EXPAND_METHOD_KW 484 +#define _MONITOR_CALL 485 +#define _MONITOR_CALL_KW 486 +#define _MONITOR_JUMP_BACKWARD 487 +#define _MONITOR_RESUME 488 #define _NOP NOP -#define _POP_CALL 488 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 -#define _POP_CALL_ONE 490 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 -#define _POP_CALL_TWO 492 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 +#define _POP_CALL 489 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 +#define _POP_CALL_ONE 491 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 +#define _POP_CALL_TWO 493 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 494 -#define _POP_JUMP_IF_TRUE 495 +#define _POP_JUMP_IF_FALSE 495 +#define _POP_JUMP_IF_TRUE 496 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 496 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 -#define _POP_TWO 498 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 +#define _POP_TOP_LOAD_CONST_INLINE 497 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 +#define _POP_TWO 499 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 500 +#define _PUSH_FRAME 501 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 501 -#define _PY_FRAME_GENERAL 502 -#define _PY_FRAME_KW 503 -#define _QUICKEN_RESUME 504 -#define _REPLACE_WITH_TRUE 505 +#define _PUSH_NULL_CONDITIONAL 502 +#define _PY_FRAME_GENERAL 503 +#define _PY_FRAME_KW 504 +#define _QUICKEN_RESUME 505 +#define _REPLACE_WITH_TRUE 506 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 506 -#define _SEND 507 -#define _SEND_GEN_FRAME 508 +#define _SAVE_RETURN_OFFSET 507 +#define _SEND 508 +#define _SEND_GEN_FRAME 509 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 509 -#define _STORE_ATTR 510 -#define _STORE_ATTR_INSTANCE_VALUE 511 -#define _STORE_ATTR_SLOT 512 -#define _STORE_ATTR_WITH_HINT 513 +#define _START_EXECUTOR 510 +#define _STORE_ATTR 511 +#define _STORE_ATTR_INSTANCE_VALUE 512 +#define _STORE_ATTR_SLOT 513 +#define _STORE_ATTR_WITH_HINT 514 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 514 -#define _STORE_FAST_0 515 -#define _STORE_FAST_1 516 -#define _STORE_FAST_2 517 -#define _STORE_FAST_3 518 -#define _STORE_FAST_4 519 -#define _STORE_FAST_5 520 -#define _STORE_FAST_6 521 -#define _STORE_FAST_7 522 +#define _STORE_FAST 515 +#define _STORE_FAST_0 516 +#define _STORE_FAST_1 517 +#define _STORE_FAST_2 518 +#define _STORE_FAST_3 519 +#define _STORE_FAST_4 520 +#define _STORE_FAST_5 521 +#define _STORE_FAST_6 522 +#define _STORE_FAST_7 523 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 523 -#define _STORE_SUBSCR 524 -#define _STORE_SUBSCR_DICT 525 -#define _STORE_SUBSCR_LIST_INT 526 +#define _STORE_SLICE 524 +#define _STORE_SUBSCR 525 +#define _STORE_SUBSCR_DICT 526 +#define _STORE_SUBSCR_LIST_INT 527 +#define _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF 528 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 527 -#define _TO_BOOL 528 +#define _TIER2_RESUME_CHECK 529 +#define _TO_BOOL 530 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 529 +#define _TO_BOOL_LIST 531 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 530 +#define _TO_BOOL_STR 532 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 531 -#define _UNPACK_SEQUENCE_LIST 532 -#define _UNPACK_SEQUENCE_TUPLE 533 -#define _UNPACK_SEQUENCE_TWO_TUPLE 534 +#define _UNPACK_SEQUENCE 533 +#define _UNPACK_SEQUENCE_LIST 534 +#define _UNPACK_SEQUENCE_TUPLE 535 +#define _UNPACK_SEQUENCE_TWO_TUPLE 536 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 534 +#define MAX_UOP_ID 536 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 978bc523999b36..1efc90238132d8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -102,6 +102,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -116,6 +117,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -348,6 +350,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", @@ -620,6 +623,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -808,6 +812,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: + return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; case _BINARY_OP_SUBSCR_STR_INT: @@ -836,6 +842,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: + return 3; case _STORE_SUBSCR_DICT: return 3; case _DELETE_SUBSCR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0d9684da51b35c..adcf95d7b763bd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -907,6 +907,32 @@ dummy_func( DECREF_INPUTS(); } + op(_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, (list_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; +#ifdef Py_GIL_DISABLED + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + DEOPT_IF(res_o == NULL); + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); +#else + DEOPT_IF(index >= PyList_GET_SIZE(list)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); +#endif + STAT_INC(BINARY_OP, hit); + INPUTS_DEAD(); + } + macro(BINARY_OP_SUBSCR_LIST_SLICE) = _GUARD_TOS_SLICE + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_SLICE; @@ -1109,6 +1135,34 @@ dummy_func( Py_DECREF(old_value); } + op(_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, (value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list)); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true); + } + STAT_INC(STORE_SUBSCR, hit); + + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + DEAD(sub_st); + DEAD(list_st); + Py_DECREF(old_value); + } + macro(STORE_SUBSCR_DICT) = _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 49e0cb2ebaba62..d536fd68c6095b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1384,6 +1384,48 @@ break; } + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_SUBSCR_LIST_SLICE: { _PyStackRef sub_st; _PyStackRef list_st; @@ -1743,6 +1785,47 @@ break; } + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + case _STORE_SUBSCR_DICT: { _PyStackRef sub; _PyStackRef dict_st; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 02022382cb4a3a..7acbf4672136e1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -448,6 +448,8 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, + [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, + [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index aa90d6b7645259..3f72d73abcfb52 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,6 +104,15 @@ dummy_func(void) { GETLOCAL(oparg) = value; } + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { + // Can't move this to the optimizer generator for now, as it requires list_st and sub_st + // to be borrowed, but not value. + // Alternatively, we could just stricten it and require all to be borrowed. + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + } + op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); } @@ -394,6 +403,14 @@ dummy_func(void) { ctx->done = true; } + op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + res = sym_new_not_null(ctx); + } + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 75bfcaf30b994a..86412bf848623f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -617,6 +617,22 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { + JitOptSymbol *sub_st; + JitOptSymbol *list_st; + JitOptSymbol *res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -759,6 +775,19 @@ } case _STORE_SUBSCR_LIST_INT: { + JitOptSymbol *sub_st; + JitOptSymbol *list_st; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From 7f90d0cf7357b2b703110c685311fbac5f317ac1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:09:28 +0800 Subject: [PATCH 03/24] Fix test, rename --- Include/internal/pycore_uop_ids.h | 10 ++++----- Include/internal/pycore_uop_metadata.h | 30 +++++++++++++------------- Lib/test/test_capi/test_opt.py | 12 +++++------ Python/bytecodes.c | 10 ++++----- Python/executor_cases.c.h | 10 ++++----- Python/optimizer_analysis.c | 10 ++++----- Python/optimizer_cases.c.h | 10 ++++----- 7 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a8e8cce09a458e..d105d7e16c740e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,24 +13,24 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF 304 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 #define _BINARY_OP_ADD_INT 305 #define _BINARY_OP_ADD_UNICODE 306 #define _BINARY_OP_EXTEND 307 #define _BINARY_OP_INPLACE_ADD_UNICODE 308 #define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF 310 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 #define _BINARY_OP_MULTIPLY_INT 311 #define _BINARY_OP_SUBSCR_CHECK_FUNC 312 #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF 316 +#define _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS 316 #define _BINARY_OP_SUBSCR_LIST_SLICE 317 #define _BINARY_OP_SUBSCR_STR_INT 318 #define _BINARY_OP_SUBSCR_TUPLE_INT 319 #define _BINARY_OP_SUBTRACT_FLOAT 320 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 321 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 321 #define _BINARY_OP_SUBTRACT_INT 322 #define _BINARY_SLICE 323 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION @@ -324,7 +324,7 @@ extern "C" { #define _STORE_SUBSCR 525 #define _STORE_SUBSCR_DICT 526 #define _STORE_SUBSCR_LIST_INT 527 -#define _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF 528 +#define _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS 528 #define _SWAP SWAP #define _TIER2_RESUME_CHECK 529 #define _TO_BOOL 530 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1efc90238132d8..dcbbdb7fc6a8fc 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,9 +92,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -102,7 +102,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -117,7 +117,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -338,24 +338,24 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", - [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF", + [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -623,7 +623,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", - [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF", + [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -792,11 +792,11 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: return 2; - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: return 2; - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: return 2; case _BINARY_OP_ADD_UNICODE: return 2; @@ -812,7 +812,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; @@ -842,7 +842,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: return 3; case _STORE_SUBSCR_DICT: return 3; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bb8090393949a7..7ccbc817bd280f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -678,7 +678,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -1707,10 +1707,10 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) def test_remove_guard_for_known_type_set(self): @@ -2231,7 +2231,7 @@ def testfunc(args): self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def global_identity(x): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index adcf95d7b763bd..77211b63ce0739 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -678,7 +678,7 @@ dummy_func( } - pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -693,7 +693,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } - pure op(_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -708,7 +708,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } - pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -907,7 +907,7 @@ dummy_func( DECREF_INPUTS(); } - op(_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, (list_st, sub_st -- res)) { + op(_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (list_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); @@ -1135,7 +1135,7 @@ dummy_func( Py_DECREF(old_value); } - op(_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, (value, list_st, sub_st -- )) { + op(_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (value, list_st, sub_st -- )) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d536fd68c6095b..aae427e4b64b96 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1043,7 +1043,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1070,7 +1070,7 @@ break; } - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1097,7 +1097,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1384,7 +1384,7 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -1785,7 +1785,7 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef value; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7acbf4672136e1..6270cd5be71a94 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -445,11 +445,11 @@ get_code_with_logging(_PyUOpInstruction *op) // TODO (gh-134584) generate most of this table automatically const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { - [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, - [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, - [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, - [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, - [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, + [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 86412bf848623f..8419eefbf7b6e0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -509,7 +509,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -518,7 +518,7 @@ break; } - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -527,7 +527,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -632,7 +632,7 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -787,7 +787,7 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From a42b43483e98039402259b6547d7fd41c76c2c7b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:51:29 +0800 Subject: [PATCH 04/24] Remove list optimizations to minimize PR --- Include/internal/pycore_uop_ids.h | 442 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 8 - Lib/test/test_capi/test_opt.py | 4 +- Python/bytecodes.c | 54 --- Python/executor_cases.c.h | 83 ----- Python/optimizer_analysis.c | 2 - Python/optimizer_bytecodes.c | 17 - Python/optimizer_cases.c.h | 29 -- 8 files changed, 222 insertions(+), 417 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d105d7e16c740e..2ba13418a18cfc 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -25,14 +25,13 @@ extern "C" { #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS 316 -#define _BINARY_OP_SUBSCR_LIST_SLICE 317 -#define _BINARY_OP_SUBSCR_STR_INT 318 -#define _BINARY_OP_SUBSCR_TUPLE_INT 319 -#define _BINARY_OP_SUBTRACT_FLOAT 320 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 321 -#define _BINARY_OP_SUBTRACT_INT 322 -#define _BINARY_SLICE 323 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -41,135 +40,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 324 -#define _CALL_BUILTIN_FAST 325 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 326 -#define _CALL_BUILTIN_O 327 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 328 -#define _CALL_KW_NON_PY 329 -#define _CALL_LEN 330 -#define _CALL_LIST_APPEND 331 -#define _CALL_METHOD_DESCRIPTOR_FAST 332 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 333 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 334 -#define _CALL_METHOD_DESCRIPTOR_O 335 -#define _CALL_NON_PY_GENERAL 336 -#define _CALL_STR_1 337 -#define _CALL_TUPLE_1 338 -#define _CALL_TYPE_1 339 -#define _CHECK_AND_ALLOCATE_OBJECT 340 -#define _CHECK_ATTR_CLASS 341 -#define _CHECK_ATTR_METHOD_LAZY_DICT 342 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 344 -#define _CHECK_FUNCTION_EXACT_ARGS 345 -#define _CHECK_FUNCTION_VERSION 346 -#define _CHECK_FUNCTION_VERSION_INLINE 347 -#define _CHECK_FUNCTION_VERSION_KW 348 -#define _CHECK_IS_NOT_PY_CALLABLE 349 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 350 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 351 -#define _CHECK_METHOD_VERSION 352 -#define _CHECK_METHOD_VERSION_KW 353 -#define _CHECK_PEP_523 354 -#define _CHECK_PERIODIC 355 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 -#define _CHECK_RECURSION_REMAINING 357 -#define _CHECK_STACK_SPACE 358 -#define _CHECK_STACK_SPACE_OPERAND 359 -#define _CHECK_VALIDITY 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _CHECK_FUNCTION 343 +#define _CHECK_FUNCTION_EXACT_ARGS 344 +#define _CHECK_FUNCTION_VERSION 345 +#define _CHECK_FUNCTION_VERSION_INLINE 346 +#define _CHECK_FUNCTION_VERSION_KW 347 +#define _CHECK_IS_NOT_PY_CALLABLE 348 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 +#define _CHECK_METHOD_VERSION 351 +#define _CHECK_METHOD_VERSION_KW 352 +#define _CHECK_PEP_523 353 +#define _CHECK_PERIODIC 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COMPARE_OP 360 +#define _COMPARE_OP_FLOAT 361 +#define _COMPARE_OP_INT 362 +#define _COMPARE_OP_STR 363 +#define _CONTAINS_OP 364 +#define _CONTAINS_OP_DICT 365 +#define _CONTAINS_OP_SET 366 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 368 +#define _CREATE_INIT_FRAME 367 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 369 +#define _DEOPT 368 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 370 -#define _DO_CALL_FUNCTION_EX 371 -#define _DO_CALL_KW 372 +#define _DO_CALL 369 +#define _DO_CALL_FUNCTION_EX 370 +#define _DO_CALL_KW 371 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 373 +#define _ERROR_POP_N 372 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 374 -#define _EXPAND_METHOD_KW 375 -#define _FATAL_ERROR 376 +#define _EXPAND_METHOD 373 +#define _EXPAND_METHOD_KW 374 +#define _FATAL_ERROR 375 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 377 -#define _FOR_ITER_GEN_FRAME 378 -#define _FOR_ITER_TIER_TWO 379 +#define _FOR_ITER 376 +#define _FOR_ITER_GEN_FRAME 377 +#define _FOR_ITER_TIER_TWO 378 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 380 -#define _GUARD_CALLABLE_ISINSTANCE 381 -#define _GUARD_CALLABLE_LEN 382 -#define _GUARD_CALLABLE_LIST_APPEND 383 -#define _GUARD_CALLABLE_STR_1 384 -#define _GUARD_CALLABLE_TUPLE_1 385 -#define _GUARD_CALLABLE_TYPE_1 386 -#define _GUARD_DORV_NO_DICT 387 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 388 -#define _GUARD_GLOBALS_VERSION 389 -#define _GUARD_IS_FALSE_POP 390 -#define _GUARD_IS_NONE_POP 391 -#define _GUARD_IS_NOT_NONE_POP 392 -#define _GUARD_IS_TRUE_POP 393 -#define _GUARD_KEYS_VERSION 394 -#define _GUARD_NOS_DICT 395 -#define _GUARD_NOS_FLOAT 396 -#define _GUARD_NOS_INT 397 -#define _GUARD_NOS_LIST 398 -#define _GUARD_NOS_NOT_NULL 399 -#define _GUARD_NOS_NULL 400 -#define _GUARD_NOS_TUPLE 401 -#define _GUARD_NOS_UNICODE 402 -#define _GUARD_NOT_EXHAUSTED_LIST 403 -#define _GUARD_NOT_EXHAUSTED_RANGE 404 -#define _GUARD_NOT_EXHAUSTED_TUPLE 405 -#define _GUARD_THIRD_NULL 406 -#define _GUARD_TOS_ANY_SET 407 -#define _GUARD_TOS_DICT 408 -#define _GUARD_TOS_FLOAT 409 -#define _GUARD_TOS_INT 410 -#define _GUARD_TOS_LIST 411 -#define _GUARD_TOS_SLICE 412 -#define _GUARD_TOS_TUPLE 413 -#define _GUARD_TOS_UNICODE 414 -#define _GUARD_TYPE_VERSION 415 -#define _GUARD_TYPE_VERSION_AND_LOCK 416 +#define _GUARD_BINARY_OP_EXTEND 379 +#define _GUARD_CALLABLE_ISINSTANCE 380 +#define _GUARD_CALLABLE_LEN 381 +#define _GUARD_CALLABLE_LIST_APPEND 382 +#define _GUARD_CALLABLE_STR_1 383 +#define _GUARD_CALLABLE_TUPLE_1 384 +#define _GUARD_CALLABLE_TYPE_1 385 +#define _GUARD_DORV_NO_DICT 386 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 +#define _GUARD_GLOBALS_VERSION 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _GUARD_IS_TRUE_POP 392 +#define _GUARD_KEYS_VERSION 393 +#define _GUARD_NOS_DICT 394 +#define _GUARD_NOS_FLOAT 395 +#define _GUARD_NOS_INT 396 +#define _GUARD_NOS_LIST 397 +#define _GUARD_NOS_NOT_NULL 398 +#define _GUARD_NOS_NULL 399 +#define _GUARD_NOS_TUPLE 400 +#define _GUARD_NOS_UNICODE 401 +#define _GUARD_NOT_EXHAUSTED_LIST 402 +#define _GUARD_NOT_EXHAUSTED_RANGE 403 +#define _GUARD_NOT_EXHAUSTED_TUPLE 404 +#define _GUARD_THIRD_NULL 405 +#define _GUARD_TOS_ANY_SET 406 +#define _GUARD_TOS_DICT 407 +#define _GUARD_TOS_FLOAT 408 +#define _GUARD_TOS_INT 409 +#define _GUARD_TOS_LIST 410 +#define _GUARD_TOS_SLICE 411 +#define _GUARD_TOS_TUPLE 412 +#define _GUARD_TOS_UNICODE 413 +#define _GUARD_TYPE_VERSION 414 +#define _GUARD_TYPE_VERSION_AND_LOCK 415 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS 418 -#define _INIT_CALL_PY_EXACT_ARGS_0 419 -#define _INIT_CALL_PY_EXACT_ARGS_1 420 -#define _INIT_CALL_PY_EXACT_ARGS_2 421 -#define _INIT_CALL_PY_EXACT_ARGS_3 422 -#define _INIT_CALL_PY_EXACT_ARGS_4 423 -#define _INSERT_NULL 424 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 +#define _INIT_CALL_PY_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS_0 418 +#define _INIT_CALL_PY_EXACT_ARGS_1 419 +#define _INIT_CALL_PY_EXACT_ARGS_2 420 +#define _INIT_CALL_PY_EXACT_ARGS_3 421 +#define _INIT_CALL_PY_EXACT_ARGS_4 422 +#define _INSERT_NULL 423 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -179,171 +178,170 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 425 +#define _IS_NONE 424 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 426 -#define _ITER_CHECK_RANGE 427 -#define _ITER_CHECK_TUPLE 428 -#define _ITER_JUMP_LIST 429 -#define _ITER_JUMP_RANGE 430 -#define _ITER_JUMP_TUPLE 431 -#define _ITER_NEXT_LIST 432 -#define _ITER_NEXT_LIST_TIER_TWO 433 -#define _ITER_NEXT_RANGE 434 -#define _ITER_NEXT_TUPLE 435 -#define _JUMP_TO_TOP 436 +#define _ITER_CHECK_LIST 425 +#define _ITER_CHECK_RANGE 426 +#define _ITER_CHECK_TUPLE 427 +#define _ITER_JUMP_LIST 428 +#define _ITER_JUMP_RANGE 429 +#define _ITER_JUMP_TUPLE 430 +#define _ITER_NEXT_LIST 431 +#define _ITER_NEXT_LIST_TIER_TWO 432 +#define _ITER_NEXT_RANGE 433 +#define _ITER_NEXT_TUPLE 434 +#define _JUMP_TO_TOP 435 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 437 -#define _LOAD_ATTR_CLASS 438 +#define _LOAD_ATTR 436 +#define _LOAD_ATTR_CLASS 437 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 439 -#define _LOAD_ATTR_METHOD_LAZY_DICT 440 -#define _LOAD_ATTR_METHOD_NO_DICT 441 -#define _LOAD_ATTR_METHOD_WITH_VALUES 442 -#define _LOAD_ATTR_MODULE 443 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 -#define _LOAD_ATTR_PROPERTY_FRAME 446 -#define _LOAD_ATTR_SLOT 447 -#define _LOAD_ATTR_WITH_HINT 448 +#define _LOAD_ATTR_INSTANCE_VALUE 438 +#define _LOAD_ATTR_METHOD_LAZY_DICT 439 +#define _LOAD_ATTR_METHOD_NO_DICT 440 +#define _LOAD_ATTR_METHOD_WITH_VALUES 441 +#define _LOAD_ATTR_MODULE 442 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 +#define _LOAD_ATTR_PROPERTY_FRAME 445 +#define _LOAD_ATTR_SLOT 446 +#define _LOAD_ATTR_WITH_HINT 447 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 449 +#define _LOAD_BYTECODE 448 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 450 -#define _LOAD_CONST_INLINE_BORROW 451 -#define _LOAD_CONST_UNDER_INLINE 452 -#define _LOAD_CONST_UNDER_INLINE_BORROW 453 +#define _LOAD_CONST_INLINE 449 +#define _LOAD_CONST_INLINE_BORROW 450 +#define _LOAD_CONST_UNDER_INLINE 451 +#define _LOAD_CONST_UNDER_INLINE_BORROW 452 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 454 -#define _LOAD_FAST_0 455 -#define _LOAD_FAST_1 456 -#define _LOAD_FAST_2 457 -#define _LOAD_FAST_3 458 -#define _LOAD_FAST_4 459 -#define _LOAD_FAST_5 460 -#define _LOAD_FAST_6 461 -#define _LOAD_FAST_7 462 +#define _LOAD_FAST 453 +#define _LOAD_FAST_0 454 +#define _LOAD_FAST_1 455 +#define _LOAD_FAST_2 456 +#define _LOAD_FAST_3 457 +#define _LOAD_FAST_4 458 +#define _LOAD_FAST_5 459 +#define _LOAD_FAST_6 460 +#define _LOAD_FAST_7 461 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 463 -#define _LOAD_FAST_BORROW_0 464 -#define _LOAD_FAST_BORROW_1 465 -#define _LOAD_FAST_BORROW_2 466 -#define _LOAD_FAST_BORROW_3 467 -#define _LOAD_FAST_BORROW_4 468 -#define _LOAD_FAST_BORROW_5 469 -#define _LOAD_FAST_BORROW_6 470 -#define _LOAD_FAST_BORROW_7 471 +#define _LOAD_FAST_BORROW 462 +#define _LOAD_FAST_BORROW_0 463 +#define _LOAD_FAST_BORROW_1 464 +#define _LOAD_FAST_BORROW_2 465 +#define _LOAD_FAST_BORROW_3 466 +#define _LOAD_FAST_BORROW_4 467 +#define _LOAD_FAST_BORROW_5 468 +#define _LOAD_FAST_BORROW_6 469 +#define _LOAD_FAST_BORROW_7 470 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 472 -#define _LOAD_GLOBAL_BUILTINS 473 -#define _LOAD_GLOBAL_MODULE 474 +#define _LOAD_GLOBAL 471 +#define _LOAD_GLOBAL_BUILTINS 472 +#define _LOAD_GLOBAL_MODULE 473 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 475 -#define _LOAD_SMALL_INT_0 476 -#define _LOAD_SMALL_INT_1 477 -#define _LOAD_SMALL_INT_2 478 -#define _LOAD_SMALL_INT_3 479 -#define _LOAD_SPECIAL 480 +#define _LOAD_SMALL_INT 474 +#define _LOAD_SMALL_INT_0 475 +#define _LOAD_SMALL_INT_1 476 +#define _LOAD_SMALL_INT_2 477 +#define _LOAD_SMALL_INT_3 478 +#define _LOAD_SPECIAL 479 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 481 +#define _MAKE_CALLARGS_A_TUPLE 480 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 482 +#define _MAKE_WARM 481 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 483 -#define _MAYBE_EXPAND_METHOD_KW 484 -#define _MONITOR_CALL 485 -#define _MONITOR_CALL_KW 486 -#define _MONITOR_JUMP_BACKWARD 487 -#define _MONITOR_RESUME 488 +#define _MAYBE_EXPAND_METHOD 482 +#define _MAYBE_EXPAND_METHOD_KW 483 +#define _MONITOR_CALL 484 +#define _MONITOR_CALL_KW 485 +#define _MONITOR_JUMP_BACKWARD 486 +#define _MONITOR_RESUME 487 #define _NOP NOP -#define _POP_CALL 489 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 -#define _POP_CALL_ONE 491 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 -#define _POP_CALL_TWO 493 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 +#define _POP_CALL 488 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 +#define _POP_CALL_ONE 490 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 +#define _POP_CALL_TWO 492 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 495 -#define _POP_JUMP_IF_TRUE 496 +#define _POP_JUMP_IF_FALSE 494 +#define _POP_JUMP_IF_TRUE 495 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 497 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 -#define _POP_TWO 499 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 +#define _POP_TOP_LOAD_CONST_INLINE 496 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 +#define _POP_TWO 498 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 501 +#define _PUSH_FRAME 500 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 502 -#define _PY_FRAME_GENERAL 503 -#define _PY_FRAME_KW 504 -#define _QUICKEN_RESUME 505 -#define _REPLACE_WITH_TRUE 506 +#define _PUSH_NULL_CONDITIONAL 501 +#define _PY_FRAME_GENERAL 502 +#define _PY_FRAME_KW 503 +#define _QUICKEN_RESUME 504 +#define _REPLACE_WITH_TRUE 505 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 507 -#define _SEND 508 -#define _SEND_GEN_FRAME 509 +#define _SAVE_RETURN_OFFSET 506 +#define _SEND 507 +#define _SEND_GEN_FRAME 508 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 510 -#define _STORE_ATTR 511 -#define _STORE_ATTR_INSTANCE_VALUE 512 -#define _STORE_ATTR_SLOT 513 -#define _STORE_ATTR_WITH_HINT 514 +#define _START_EXECUTOR 509 +#define _STORE_ATTR 510 +#define _STORE_ATTR_INSTANCE_VALUE 511 +#define _STORE_ATTR_SLOT 512 +#define _STORE_ATTR_WITH_HINT 513 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 515 -#define _STORE_FAST_0 516 -#define _STORE_FAST_1 517 -#define _STORE_FAST_2 518 -#define _STORE_FAST_3 519 -#define _STORE_FAST_4 520 -#define _STORE_FAST_5 521 -#define _STORE_FAST_6 522 -#define _STORE_FAST_7 523 +#define _STORE_FAST 514 +#define _STORE_FAST_0 515 +#define _STORE_FAST_1 516 +#define _STORE_FAST_2 517 +#define _STORE_FAST_3 518 +#define _STORE_FAST_4 519 +#define _STORE_FAST_5 520 +#define _STORE_FAST_6 521 +#define _STORE_FAST_7 522 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 524 -#define _STORE_SUBSCR 525 -#define _STORE_SUBSCR_DICT 526 -#define _STORE_SUBSCR_LIST_INT 527 -#define _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS 528 +#define _STORE_SLICE 523 +#define _STORE_SUBSCR 524 +#define _STORE_SUBSCR_DICT 525 +#define _STORE_SUBSCR_LIST_INT 526 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 529 -#define _TO_BOOL 530 +#define _TIER2_RESUME_CHECK 527 +#define _TO_BOOL 528 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 531 +#define _TO_BOOL_LIST 529 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 532 +#define _TO_BOOL_STR 530 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 533 -#define _UNPACK_SEQUENCE_LIST 534 -#define _UNPACK_SEQUENCE_TUPLE 535 -#define _UNPACK_SEQUENCE_TWO_TUPLE 536 +#define _UNPACK_SEQUENCE 531 +#define _UNPACK_SEQUENCE_LIST 532 +#define _UNPACK_SEQUENCE_TUPLE 533 +#define _UNPACK_SEQUENCE_TWO_TUPLE 534 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 536 +#define MAX_UOP_ID 534 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index dcbbdb7fc6a8fc..6b298211949cc9 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -102,7 +102,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -117,7 +116,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -350,7 +348,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", - [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", @@ -623,7 +620,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", - [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -812,8 +808,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: - return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; case _BINARY_OP_SUBSCR_STR_INT: @@ -842,8 +836,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: - return 3; case _STORE_SUBSCR_DICT: return 3; case _DELETE_SUBSCR: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 7ccbc817bd280f..8c6b5639e86624 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1707,10 +1707,10 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) def test_remove_guard_for_known_type_set(self): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 77211b63ce0739..58453549381dd9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -907,32 +907,6 @@ dummy_func( DECREF_INPUTS(); } - op(_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (list_st, sub_st -- res)) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; -#ifdef Py_GIL_DISABLED - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - DEOPT_IF(res_o == NULL); - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); -#else - DEOPT_IF(index >= PyList_GET_SIZE(list)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); -#endif - STAT_INC(BINARY_OP, hit); - INPUTS_DEAD(); - } - macro(BINARY_OP_SUBSCR_LIST_SLICE) = _GUARD_TOS_SLICE + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_SLICE; @@ -1135,34 +1109,6 @@ dummy_func( Py_DECREF(old_value); } - op(_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (value, list_st, sub_st -- )) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list)); - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - DEOPT_IF(true); - } - STAT_INC(STORE_SUBSCR, hit); - - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - DEAD(sub_st); - DEAD(list_st); - Py_DECREF(old_value); - } - macro(STORE_SUBSCR_DICT) = _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index aae427e4b64b96..c9fed6974b560f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1384,48 +1384,6 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _BINARY_OP_SUBSCR_LIST_SLICE: { _PyStackRef sub_st; _PyStackRef list_st; @@ -1785,47 +1743,6 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef value; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - break; - } - case _STORE_SUBSCR_DICT: { _PyStackRef sub; _PyStackRef dict_st; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6270cd5be71a94..72e522da1f80d0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -448,8 +448,6 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, - [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, - [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3f72d73abcfb52..aa90d6b7645259 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,15 +104,6 @@ dummy_func(void) { GETLOCAL(oparg) = value; } - op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { - // Can't move this to the optimizer generator for now, as it requires list_st and sub_st - // to be borrowed, but not value. - // Alternatively, we could just stricten it and require all to be borrowed. - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - } - op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); } @@ -403,14 +394,6 @@ dummy_func(void) { ctx->done = true; } - op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { - // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - res = sym_new_not_null(ctx); - } - op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8419eefbf7b6e0..3ff4e056e7dac8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -617,22 +617,6 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *list_st; - JitOptSymbol *res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - res = sym_new_not_null(ctx); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -775,19 +759,6 @@ } case _STORE_SUBSCR_LIST_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *list_st; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From f4567404f5c5268d4756bb081f45ffc343f06516 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 14:54:13 +0000 Subject: [PATCH 05/24] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst new file mode 100644 index 00000000000000..5f9e1553ae7ca5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst @@ -0,0 +1 @@ +Add a reference count elimination pass to the JIT compiler. Patch by Ken Jin. From 5c429b60831de05ecb3ba21e9ec3dcee50de724f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 02:50:28 +0800 Subject: [PATCH 06/24] Rename things to make things clearer --- Include/internal/pycore_optimizer.h | 18 ++++++++--------- Python/optimizer_analysis.c | 6 +++--- Python/optimizer_bytecodes.c | 14 ++++++------- Python/optimizer_cases.c.h | 14 ++++++------- Python/optimizer_symbols.c | 31 +++++++++++++++-------------- 5 files changed, 41 insertions(+), 42 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e8b61096f477a1..99b288d5e4efd2 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,8 +180,8 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define DONT_SKIP_REFCOUNT 0 -#define SKIP_REFCOUNT 1 +#define WE_MIGHT_BE_THE_ONLY_STRONG_REF 0 +#define SOMEONE_ELSE_HAS_A_VALID_STRONG_REF 1 typedef struct _jit_opt_known_class { struct { @@ -231,11 +231,9 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object skips refcount on the stack - // (using the _PyStackRef API), or not. - // 0 - normal refcounting - // 1 - skip refcounting - int8_t skip_refcount; + // Whether this object has a strong reference + // being held to it by someone else. + int8_t strong_ref_held_by_someone_else; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -308,9 +306,9 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); -extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0ee45526a68406..0fd09cbe81f8f0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,9 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount -#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount -#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount +#define sym_set_strong_ref_is_held_by_someone_else _Py_uop_sym_set_strong_ref_is_held_by_someone_else +#define sym_set_strong_ref_might_be_this_sym _Py_uop_sym_set_strong_ref_might_be_this_sym +#define sym_is_strong_ref_held_by_someone_else _Py_uop_sym_is_strong_ref_held_by_someone_else static int optimize_to_bool( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 41cde50aed1206..d2583de953b132 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,24 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + sym_set_strong_ref_is_held_by_someone_else(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_STORE_FAST, (value --)) { @@ -301,7 +301,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -325,7 +325,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -349,7 +349,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e6d191889b4476..b6908029baac8a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,7 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -41,7 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -51,7 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + sym_set_strong_ref_is_held_by_someone_else(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -63,7 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -444,7 +444,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -476,7 +476,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -508,7 +508,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 68f1ecd1d316a4..3cbad86eb61df2 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,7 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; - self->skip_refcount = DONT_SKIP_REFCOUNT; + self->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; return self; } @@ -109,7 +109,7 @@ static void make_const(JitOptSymbol *sym, PyObject *val) sym->value.value = Py_NewRef(val); // Constants don't need to be refcounted, as they are always // kept alive by co_consts. - sym->skip_refcount = SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } static inline void @@ -386,22 +386,23 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) { - sym->skip_refcount = SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } void -_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym) { - sym->skip_refcount = DONT_SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; } bool -_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) { - assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); - return sym->skip_refcount == SKIP_REFCOUNT; + assert(sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF || + sym->strong_ref_held_by_someone_else == WE_MIGHT_BE_THE_ONLY_STRONG_REF); + return sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } @@ -794,7 +795,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -843,7 +844,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -876,12 +877,12 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_skip_refcount(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_strong_ref_is_held_by_someone_else(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From 15351331857d66fbdc965c8faa4f46ecb32aefa8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 02:52:27 +0800 Subject: [PATCH 07/24] Revert "Rename things to make things clearer" This reverts commit 5c429b60831de05ecb3ba21e9ec3dcee50de724f. --- Include/internal/pycore_optimizer.h | 18 +++++++++-------- Python/optimizer_analysis.c | 6 +++--- Python/optimizer_bytecodes.c | 14 ++++++------- Python/optimizer_cases.c.h | 14 ++++++------- Python/optimizer_symbols.c | 31 ++++++++++++++--------------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 99b288d5e4efd2..e8b61096f477a1 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,8 +180,8 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define WE_MIGHT_BE_THE_ONLY_STRONG_REF 0 -#define SOMEONE_ELSE_HAS_A_VALID_STRONG_REF 1 +#define DONT_SKIP_REFCOUNT 0 +#define SKIP_REFCOUNT 1 typedef struct _jit_opt_known_class { struct { @@ -231,9 +231,11 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object has a strong reference - // being held to it by someone else. - int8_t strong_ref_held_by_someone_else; + // Whether this object skips refcount on the stack + // (using the _PyStackRef API), or not. + // 0 - normal refcounting + // 1 - skip refcounting + int8_t skip_refcount; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -306,9 +308,9 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); -extern void _Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0fd09cbe81f8f0..0ee45526a68406 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,9 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_strong_ref_is_held_by_someone_else _Py_uop_sym_set_strong_ref_is_held_by_someone_else -#define sym_set_strong_ref_might_be_this_sym _Py_uop_sym_set_strong_ref_might_be_this_sym -#define sym_is_strong_ref_held_by_someone_else _Py_uop_sym_is_strong_ref_held_by_someone_else +#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount +#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount +#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount static int optimize_to_bool( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d2583de953b132..41cde50aed1206 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,24 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); - sym_set_strong_ref_is_held_by_someone_else(ctx, value); + sym_set_skip_refcount(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -301,7 +301,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -325,7 +325,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -349,7 +349,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b6908029baac8a..e6d191889b4476 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,7 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -41,7 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -51,7 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_strong_ref_is_held_by_someone_else(ctx, value); + sym_set_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -63,7 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -444,7 +444,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -476,7 +476,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -508,7 +508,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 3cbad86eb61df2..68f1ecd1d316a4 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,7 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; - self->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; + self->skip_refcount = DONT_SKIP_REFCOUNT; return self; } @@ -109,7 +109,7 @@ static void make_const(JitOptSymbol *sym, PyObject *val) sym->value.value = Py_NewRef(val); // Constants don't need to be refcounted, as they are always // kept alive by co_consts. - sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -386,23 +386,22 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + sym->skip_refcount = SKIP_REFCOUNT; } void -_Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - sym->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; + sym->skip_refcount = DONT_SKIP_REFCOUNT; } bool -_Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - assert(sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF || - sym->strong_ref_held_by_someone_else == WE_MIGHT_BE_THE_ONLY_STRONG_REF); - return sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); + return sym->skip_refcount == SKIP_REFCOUNT; } @@ -795,7 +794,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "top is refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -844,7 +843,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "42 isn't refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -877,12 +876,12 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_strong_ref_is_held_by_someone_else(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should not be refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_skip_refcount(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "None should not be refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From 8f6206795863595a440c2647db77f09aec72fab2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:26:12 +0800 Subject: [PATCH 08/24] Massive refactor from JitOptSymbol to JitRef --- Include/internal/pycore_optimizer.h | 130 +- Python/optimizer_analysis.c | 71 +- Python/optimizer_bytecodes.c | 566 +++++---- Python/optimizer_cases.c.h | 1172 +++++++++--------- Python/optimizer_symbols.c | 402 +++--- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/optimizer_generator.py | 14 +- 7 files changed, 1189 insertions(+), 1168 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e8b61096f477a1..78cea6198d9767 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_typedefs.h" // _PyInterpreterFrame #include "pycore_uop_ids.h" +#include "pycore_stackref.h" #include @@ -180,13 +181,10 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define DONT_SKIP_REFCOUNT 0 -#define SKIP_REFCOUNT 1 typedef struct _jit_opt_known_class { struct { uint8_t tag; - uint8_t skip_refcount; }; uint32_t version; PyTypeObject *type; @@ -195,7 +193,6 @@ typedef struct _jit_opt_known_class { typedef struct _jit_opt_known_version { struct { uint8_t tag; - uint8_t skip_refcount; }; uint32_t version; } JitOptKnownVersion; @@ -203,7 +200,6 @@ typedef struct _jit_opt_known_version { typedef struct _jit_opt_known_value { struct { uint8_t tag; - uint8_t skip_refcount; }; PyObject *value; } JitOptKnownValue; @@ -213,7 +209,6 @@ typedef struct _jit_opt_known_value { typedef struct _jit_opt_tuple { struct { uint8_t tag; - uint8_t skip_refcount; }; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; @@ -222,7 +217,6 @@ typedef struct _jit_opt_tuple { typedef struct { struct { uint8_t tag; - uint8_t skip_refcount; }; bool invert; uint16_t value; @@ -231,11 +225,6 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object skips refcount on the stack - // (using the _PyStackRef API), or not. - // 0 - normal refcounting - // 1 - skip refcounting - int8_t skip_refcount; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -245,15 +234,65 @@ typedef union _jit_opt_symbol { } JitOptSymbol; +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +#define JIT_BITS_TO_PTR(REF) ((JitOptSymbol *)((REF).bits)) +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) + +static inline JitOptSymbol * +PyJitRef_AsSymbolBorrow(JitOptRef ref) +{ + return JIT_BITS_TO_PTR_MASKED(ref); +} + +bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); + +static inline JitOptRef +PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +{ + if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + } + return (JitOptRef){.bits=(uintptr_t)sym}; +} + +static inline JitOptRef +PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) +{ + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; +} + +static inline JitOptRef +PyJitRef_Borrow(JitOptRef ref) +{ + return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; +} + +static const JitOptRef PyJitRef_NULL = { .bits = PyStackRef_NULL_BITS }; + +static inline bool +PyJitRef_IsNull(JitOptRef ref) +{ + return ref.bits == PyStackRef_NULL_BITS; +} + +static inline int +PyJitRef_IsBorrowed(JitOptRef ref) +{ + return (ref.bits & Py_TAG_REFCNT) == Py_TAG_REFCNT; +} struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - JitOptSymbol **stack_pointer; - JitOptSymbol **stack; - JitOptSymbol **locals; + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -276,41 +315,36 @@ typedef struct _JitOptContext { // Arena for the symbolic types. ty_arena t_arena; - JitOptSymbol **n_consumed; - JitOptSymbol **limit; - JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + JitOptRef *n_consumed; + JitOptRef *limit; + JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_ref_is_null(JitOptRef sym); +extern bool _Py_uop_ref_is_not_null(JitOptRef sym); +extern bool _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_ref_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx); -extern bool _Py_uop_sym_has_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); -extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); -extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); -extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); - -extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx); +extern bool _Py_uop_ref_has_type(JitOptRef sym); +extern bool _Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_ref_is_bottom(JitOptRef sym); +extern int _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_ref_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); +extern int _Py_uop_ref_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); @@ -319,7 +353,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len); extern int _Py_uop_frame_pop(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0ee45526a68406..874d4482f24f5f 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -315,54 +315,51 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_has_type _Py_uop_sym_has_type -#define sym_get_type _Py_uop_sym_get_type -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_is_bottom _Py_uop_sym_is_bottom -#define sym_truthiness _Py_uop_sym_truthiness +#define ref_is_not_null _Py_uop_ref_is_not_null +#define ref_is_const _Py_uop_ref_is_const +#define ref_get_const _Py_uop_ref_get_const +#define ref_new_unknown _Py_uop_ref_new_unknown +#define ref_new_not_null _Py_uop_ref_new_not_null +#define ref_new_type _Py_uop_ref_new_type +#define ref_is_null _Py_uop_ref_is_null +#define ref_new_const _Py_uop_ref_new_const +#define ref_new_null _Py_uop_ref_new_null +#define ref_has_type _Py_uop_ref_has_type +#define ref_get_type _Py_uop_ref_get_type +#define ref_matches_type _Py_uop_ref_matches_type +#define ref_matches_type_version _Py_uop_ref_matches_type_version +#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) +#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) +#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) +#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) +#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) +#define ref_is_bottom _Py_uop_ref_is_bottom +#define ref_truthiness _Py_uop_ref_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define sym_new_tuple _Py_uop_sym_new_tuple -#define sym_tuple_getitem _Py_uop_sym_tuple_getitem -#define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal -#define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount -#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount -#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount +#define ref_new_tuple _Py_uop_ref_new_tuple +#define ref_tuple_getitem _Py_uop_ref_tuple_getitem +#define ref_tuple_length _Py_uop_ref_tuple_length +#define ref_is_immortal _Py_uop_ref_is_immortal +#define ref_new_truthiness _Py_uop_ref_new_truthiness static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr) + JitOptRef value, + JitOptRef *result_ptr) { - if (sym_matches_type(value, &PyBool_Type)) { + if (ref_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); *result_ptr = value; return 1; } - int truthiness = sym_truthiness(ctx, value); + int truthiness = ref_truthiness(ctx, value); if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - *result_ptr = sym_new_const(ctx, load); + *result_ptr = ref_new_const(ctx, load); return 1; } return 0; @@ -378,7 +375,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) } } -static JitOptSymbol * +static JitOptRef lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) @@ -389,10 +386,10 @@ lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); - return sym_new_const(ctx, lookup); + return ref_new_const(ctx, lookup); } } - return sym_new_not_null(ctx); + return ref_new_not_null(ctx); } /* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a @@ -487,7 +484,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 41cde50aed1206..333971a2e02828 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -9,32 +9,32 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; /* Shortened forms for convenience */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_get_type _Py_uop_sym_get_type -#define sym_has_type _Py_uop_sym_has_type -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_is_bottom _Py_uop_sym_is_bottom +#define ref_is_not_null _Py_uop_ref_is_not_null +#define ref_is_const _Py_uop_ref_is_const +#define ref_get_const _Py_uop_ref_get_const +#define ref_new_unknown _Py_uop_ref_new_unknown +#define ref_new_not_null _Py_uop_ref_new_not_null +#define ref_new_type _Py_uop_ref_new_type +#define ref_is_null _Py_uop_ref_is_null +#define ref_new_const _Py_uop_ref_new_const +#define ref_new_null _Py_uop_ref_new_null +#define ref_matches_type _Py_uop_ref_matches_type +#define ref_matches_type_version _Py_uop_ref_matches_type_version +#define ref_get_type _Py_uop_ref_get_type +#define ref_has_type _Py_uop_ref_has_type +#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) +#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) +#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) +#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) +#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) +#define ref_is_bottom _Py_uop_ref_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define sym_new_tuple _Py_uop_sym_new_tuple -#define sym_tuple_getitem _Py_uop_sym_tuple_getitem -#define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal -#define sym_new_truthiness _Py_uop_sym_new_truthiness +#define ref_new_tuple _Py_uop_ref_new_tuple +#define ref_tuple_getitem _Py_uop_ref_tuple_getitem +#define ref_tuple_length _Py_uop_ref_tuple_length +#define ref_is_immortal _Py_uop_ref_is_immortal +#define ref_new_truthiness _Py_uop_ref_new_truthiness extern int optimize_to_bool( @@ -77,27 +77,23 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (ref_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = ref_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -105,38 +101,38 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); + res = ref_new_null(ctx); } op(_GUARD_TOS_INT, (value -- value)) { - if (sym_matches_type(value, &PyLong_Type)) { + if (ref_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + ref_set_type(value, &PyLong_Type); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyLong_Type)) { + if (ref_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + ref_set_type(left, &PyLong_Type); } op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == sym_get_const(ctx, owner)) { + if (type == ref_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - sym_set_const(owner, type); + ref_set_const(owner, type); } } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); - if (sym_matches_type_version(owner, type_version)) { + if (ref_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this @@ -148,7 +144,7 @@ dummy_func(void) { // if it wasn't this means that the type version was previously set to something else // and we set the owner to bottom, so we don't need to add a watcher because we must have // already added one earlier. - if (sym_set_type_version(owner, type_version)) { + if (ref_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -158,27 +154,27 @@ dummy_func(void) { } op(_GUARD_TOS_FLOAT, (value -- value)) { - if (sym_matches_type(value, &PyFloat_Type)) { + if (ref_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyFloat_Type); + ref_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyFloat_Type)) { + if (ref_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyFloat_Type); + ref_set_type(left, &PyFloat_Type); } op(_BINARY_OP, (lhs, rhs -- res)) { - bool lhs_int = sym_matches_type(lhs, &PyLong_Type); - bool rhs_int = sym_matches_type(rhs, &PyLong_Type); - bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); - bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); + bool lhs_int = ref_matches_type(lhs, &PyLong_Type); + bool rhs_int = ref_matches_type(rhs, &PyLong_Type); + bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); + bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { // This one's fun... the *type* of the result depends on the @@ -195,195 +191,195 @@ dummy_func(void) { if (rhs_float) { // Case D, E, F, or G... can't know without the sign of the LHS // or whether the RHS is whole, which isn't worth the effort: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (lhs_float) { // Case C: - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, rhs)) { + else if (!ref_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { + else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { // Case B: - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else { // Case A: - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + JitOptRef res; + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; @@ -395,109 +391,109 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(ref_matches_type(tuple_st, &PyTuple_Type)); + if (ref_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); + long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + int tuple_length = ref_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } else { assert(index < tuple_length); - res = sym_tuple_getitem(ctx, tuple_st, index); + res = ref_tuple_getitem(ctx, tuple_st, index); } } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } } op(_TO_BOOL, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyBool_Type); + value = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_INT, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_type(value, &PyLong_Type); - res = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyLong_Type); + res = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_LIST, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); + ref_set_const(value, Py_None); + res = ref_new_const(ctx, Py_False); } } op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyUnicode_Type)) { + if (ref_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyUnicode_Type); + ref_set_type(nos, &PyUnicode_Type); } op(_GUARD_TOS_UNICODE, (value -- value)) { - if (sym_matches_type(value, &PyUnicode_Type)) { + if (ref_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyUnicode_Type); + ref_set_type(value, &PyUnicode_Type); } op(_TO_BOOL_STR, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } } op(_UNARY_NOT, (value -- res)) { - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); + ref_set_type(value, &PyBool_Type); + res = ref_new_truthiness(ctx, value, false); } op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_sym_new_not_null(ctx); + res = _Py_uop_ref_new_not_null(ctx); } } op(_COMPARE_OP_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), + ref_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -505,42 +501,42 @@ dummy_func(void) { assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); + res = ref_new_const(ctx, tmp); Py_DECREF(tmp); } else { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } } op(_COMPARE_OP_FLOAT, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_SET, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_DICT, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -548,35 +544,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -585,23 +581,23 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)offset; } op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; - if (sym_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); + attr = PyJitRef_NULL; + if (ref_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); @@ -609,20 +605,20 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + attr = ref_new_const(ctx, res); } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); } } op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = sym_new_null(ctx); + null[0] = ref_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -631,25 +627,25 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = sym_new_unknown(ctx); + self_or_null[0] = ref_new_unknown(ctx); } } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)hint; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)index; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -658,7 +654,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -667,7 +663,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -676,7 +672,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -686,7 +682,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -696,7 +692,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -711,26 +707,26 @@ dummy_func(void) { } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(ctx, callable))); + if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(ref_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); } - sym_set_type(callable, &PyFunction_Type); + ref_set_type(callable, &PyFunction_Type); } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(ctx, callable)) { - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); + assert(ref_matches_type(callable, &PyFunction_Type)); + if (ref_is_const(ctx, callable)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -738,8 +734,8 @@ dummy_func(void) { } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + ref_set_null(null); + ref_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -756,13 +752,13 @@ dummy_func(void) { assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (ref_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -772,8 +768,8 @@ dummy_func(void) { op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -796,8 +792,8 @@ dummy_func(void) { op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { @@ -806,7 +802,7 @@ dummy_func(void) { } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + JitOptRef temp = retval; DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; @@ -835,7 +831,7 @@ dummy_func(void) { ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -853,7 +849,7 @@ dummy_func(void) { } op(_YIELD_VALUE, (unused -- value)) { - value = sym_new_unknown(ctx); + value = ref_new_unknown(ctx); } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { @@ -917,7 +913,7 @@ dummy_func(void) { (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } } @@ -926,40 +922,40 @@ dummy_func(void) { /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { - next = sym_new_type(ctx, &PyLong_Type); + next = ref_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + if (ref_has_type(arg)) { + res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } } op(_CALL_STR_1, (unused, unused, arg -- res)) { - if (sym_matches_type(arg, &PyUnicode_Type)) { + if (ref_matches_type(arg, &PyUnicode_Type)) { // e.g. str('foo') or str(foo) where foo is known to be a string res = arg; } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } } op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { // the result is always a bool, but sometimes we can // narrow it down to True or False - res = sym_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = sym_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); - if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + res = ref_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = ref_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); + if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { // isinstance(inst, cls) where both inst and cls have // known types, meaning we can deduce either True or False @@ -968,50 +964,50 @@ dummy_func(void) { if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - sym_set_const(res, out); + ref_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } op(_GUARD_IS_TRUE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - sym_set_const(flag, Py_True); + ref_set_const(flag, Py_True); } op(_GUARD_IS_FALSE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - sym_set_const(flag, Py_False); + ref_set_const(flag, Py_False); } op(_GUARD_IS_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(val, Py_None); + ref_set_const(val, Py_None); } op(_GUARD_IS_NOT_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -1025,13 +1021,13 @@ dummy_func(void) { } op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[0] = sym_new_null(ctx); + method_and_self[0] = ref_new_null(ctx); method_and_self[1] = self; } op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { - method_and_self[0] = sym_new_not_null(ctx); - method_and_self[1] = sym_new_unknown(ctx); + method_and_self[0] = ref_new_not_null(ctx); + method_and_self[1] = ref_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { @@ -1045,154 +1041,154 @@ dummy_func(void) { op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = sym_new_const(ctx, Py_True); + res = ref_new_const(ctx, Py_True); } op(_BUILD_TUPLE, (values[oparg] -- tup)) { - tup = sym_new_tuple(ctx, oparg, values); + tup = ref_new_tuple(ctx, oparg, values); } op(_BUILD_LIST, (values[oparg] -- list)) { - list = sym_new_type(ctx, &PyList_Type); + list = ref_new_type(ctx, &PyList_Type); } op(_BUILD_SLICE, (args[oparg] -- slice)) { - slice = sym_new_type(ctx, &PySlice_Type); + slice = ref_new_type(ctx, &PySlice_Type); } op(_BUILD_MAP, (values[oparg*2] -- map)) { - map = sym_new_type(ctx, &PyDict_Type); + map = ref_new_type(ctx, &PyDict_Type); } op(_BUILD_STRING, (pieces[oparg] -- str)) { - str = sym_new_type(ctx, &PyUnicode_Type); + str = ref_new_type(ctx, &PyUnicode_Type); } op(_BUILD_SET, (values[oparg] -- set)) { - set = sym_new_type(ctx, &PySet_Type); + set = ref_new_type(ctx, &PySet_Type); } op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { - val0 = sym_tuple_getitem(ctx, seq, 0); - val1 = sym_tuple_getitem(ctx, seq, 1); + val0 = ref_tuple_getitem(ctx, seq, 0); + val1 = ref_tuple_getitem(ctx, seq, 1); } op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); } } op(_CALL_TUPLE_1, (callable, null, arg -- res)) { - if (sym_matches_type(arg, &PyTuple_Type)) { + if (ref_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple res = arg; } else { - res = sym_new_type(ctx, &PyTuple_Type); + res = ref_new_type(ctx, &PyTuple_Type); } } op(_GUARD_TOS_LIST, (tos -- tos)) { - if (sym_matches_type(tos, &PyList_Type)) { + if (ref_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyList_Type); + ref_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyList_Type)) { + if (ref_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyList_Type); + ref_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { - if (sym_matches_type(tos, &PyTuple_Type)) { + if (ref_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyTuple_Type); + ref_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyTuple_Type)) { + if (ref_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyTuple_Type); + ref_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { - if (sym_matches_type(tos, &PyDict_Type)) { + if (ref_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyDict_Type); + ref_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyDict_Type)) { + if (ref_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyDict_Type); + ref_set_type(nos, &PyDict_Type); } op(_GUARD_TOS_ANY_SET, (tos -- tos)) { - if (sym_matches_type(tos, &PySet_Type) || - sym_matches_type(tos, &PyFrozenSet_Type)) + if (ref_matches_type(tos, &PySet_Type) || + ref_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - if (sym_is_not_null(nos)) { + if (ref_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_non_null(nos); + ref_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyType_Type); + ref_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyTuple_Type); + ref_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyUnicode_Type); + ref_set_const(callable, (PyObject *)&PyUnicode_Type); } op(_CALL_LEN, (unused, unused, unused -- res)) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } op(_GET_LEN, (obj -- obj, len)) { - int tuple_length = sym_tuple_length(obj); + int tuple_length = ref_tuple_length(obj); if (tuple_length == -1) { - len = sym_new_type(ctx, &PyLong_Type); + len = ref_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1203,33 +1199,33 @@ dummy_func(void) { if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = sym_new_const(ctx, temp); + len = ref_new_const(ctx, temp); Py_DECREF(temp); } } op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (sym_get_const(ctx, callable) == len) { + if (ref_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, len); + ref_set_const(callable, len); } op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (sym_get_const(ctx, callable) == isinstance) { + if (ref_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, isinstance); + ref_set_const(callable, isinstance); } op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (sym_get_const(ctx, callable) == list_append) { + if (ref_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, list_append); + ref_set_const(callable, list_append); } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e6d191889b4476..f59a9cbd6d217a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -26,12 +26,11 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - if (sym_is_null(value)) { + if (ref_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -39,9 +38,8 @@ } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -49,9 +47,8 @@ } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -59,11 +56,10 @@ } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = ref_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -71,10 +67,10 @@ } case _LOAD_CONST: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -82,12 +78,12 @@ } case _LOAD_SMALL_INT: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -95,7 +91,7 @@ } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -116,8 +112,8 @@ } case _PUSH_NULL: { - JitOptSymbol *res; - res = sym_new_null(ctx); + JitOptRef res; + res = ref_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -137,8 +133,8 @@ } case _END_SEND: { - JitOptSymbol *val; - val = sym_new_not_null(ctx); + JitOptRef val; + val = ref_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -146,76 +142,76 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); + ref_set_type(value, &PyBool_Type); + res = ref_new_truthiness(ctx, value, false); stack_pointer[-1] = res; break; } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyBool_Type); + value = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_type(value, &PyLong_Type); - res = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyLong_Type); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyList_Type)) { + if (ref_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyList_Type); + ref_set_type(nos, &PyList_Type); break; } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyList_Type)) { + if (ref_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyList_Type); + ref_set_type(tos, &PyList_Type); break; } @@ -224,119 +220,119 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); + ref_set_const(value, Py_None); + res = ref_new_const(ctx, Py_False); } stack_pointer[-1] = res; break; } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyUnicode_Type)) { + if (ref_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyUnicode_Type); + ref_set_type(nos, &PyUnicode_Type); break; } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyUnicode_Type)) { + if (ref_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyUnicode_Type); + ref_set_type(value, &PyUnicode_Type); break; } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = sym_new_const(ctx, Py_True); + res = ref_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_NOS_INT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { + if (ref_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + ref_set_type(left, &PyLong_Type); break; } case _GUARD_TOS_INT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyLong_Type)) { + if (ref_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + ref_set_type(value, &PyLong_Type); break; } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -344,27 +340,27 @@ } case _BINARY_OP_ADD_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -372,27 +368,27 @@ } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -400,51 +396,51 @@ } case _GUARD_NOS_FLOAT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type)) { + if (ref_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyFloat_Type); + ref_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyFloat_Type)) { + if (ref_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyFloat_Type); + ref_set_type(value, &PyFloat_Type); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -452,31 +448,31 @@ } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -484,31 +480,31 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -516,8 +512,8 @@ } case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -525,8 +521,8 @@ } case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -534,8 +530,8 @@ } case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -543,26 +539,26 @@ } case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -570,23 +566,23 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + JitOptRef res; + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; @@ -599,8 +595,8 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -608,8 +604,8 @@ } case _BINARY_SLICE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -623,8 +619,8 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -632,8 +628,8 @@ } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -641,8 +637,8 @@ } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyUnicode_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -650,47 +646,47 @@ } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyTuple_Type)) { + if (ref_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyTuple_Type); + ref_set_type(nos, &PyTuple_Type); break; } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyTuple_Type)) { + if (ref_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyTuple_Type); + ref_set_type(tos, &PyTuple_Type); break; } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(ref_matches_type(tuple_st, &PyTuple_Type)); + if (ref_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); + long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + int tuple_length = ref_tuple_length(tuple_st); if (tuple_length == -1) { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } else { assert(index < tuple_length); - res = sym_tuple_getitem(ctx, tuple_st, index); + res = ref_tuple_getitem(ctx, tuple_st, index); } } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -699,28 +695,28 @@ } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyDict_Type)) { + if (ref_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyDict_Type); + ref_set_type(nos, &PyDict_Type); break; } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyDict_Type)) { + if (ref_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyDict_Type); + ref_set_type(tos, &PyDict_Type); break; } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -728,8 +724,8 @@ } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; - getitem = sym_new_not_null(ctx); + JitOptRef getitem; + getitem = ref_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -740,7 +736,7 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3] = (JitOptSymbol *)new_frame; + stack_pointer[-3].bits = (uintptr_t)new_frame; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -783,15 +779,15 @@ } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -799,10 +795,10 @@ } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -826,15 +822,15 @@ } case _GET_AITER: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; - awaitable = sym_new_not_null(ctx); + JitOptRef awaitable; + awaitable = ref_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -842,8 +838,8 @@ } case _GET_AWAITABLE: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -854,13 +850,13 @@ _Py_UOpsAbstractFrame *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)gen_frame; + stack_pointer[-1].bits = (uintptr_t)gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *value; - value = sym_new_unknown(ctx); + JitOptRef value; + value = ref_new_unknown(ctx); stack_pointer[-1] = value; break; } @@ -872,8 +868,8 @@ } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -881,8 +877,8 @@ } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; - bc = sym_new_not_null(ctx); + JitOptRef bc; + bc = ref_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -900,13 +896,13 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[-1 + oparg]; (void)top; for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -914,12 +910,12 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; - val0 = sym_tuple_getitem(ctx, seq, 0); - val1 = sym_tuple_getitem(ctx, seq, 1); + val0 = ref_tuple_getitem(ctx, seq, 0); + val1 = ref_tuple_getitem(ctx, seq, 1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -928,12 +924,12 @@ } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -941,10 +937,10 @@ } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_not_null(ctx); + values[_i] = ref_new_not_null(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -952,14 +948,14 @@ } case _UNPACK_EX: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -989,8 +985,8 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; - locals = sym_new_not_null(ctx); + JitOptRef locals; + locals = ref_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1000,8 +996,8 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; - v = sym_new_not_null(ctx); + JitOptRef v; + v = ref_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1009,20 +1005,20 @@ } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; - res[0] = sym_new_not_null(ctx); + res[0] = ref_new_not_null(ctx); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = sym_new_null(ctx); + null[0] = ref_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1037,8 +1033,8 @@ } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1046,8 +1042,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1067,15 +1063,15 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1093,8 +1089,8 @@ } case _BUILD_STRING: { - JitOptSymbol *str; - str = sym_new_type(ctx, &PyUnicode_Type); + JitOptRef str; + str = ref_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1102,8 +1098,8 @@ } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; - interpolation = sym_new_not_null(ctx); + JitOptRef interpolation; + interpolation = ref_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1111,8 +1107,8 @@ } case _BUILD_TEMPLATE: { - JitOptSymbol *template; - template = sym_new_not_null(ctx); + JitOptRef template; + template = ref_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1120,10 +1116,10 @@ } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; - tup = sym_new_tuple(ctx, oparg, values); + tup = ref_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1131,8 +1127,8 @@ } case _BUILD_LIST: { - JitOptSymbol *list; - list = sym_new_type(ctx, &PyList_Type); + JitOptRef list; + list = ref_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1152,8 +1148,8 @@ } case _BUILD_SET: { - JitOptSymbol *set; - set = sym_new_type(ctx, &PySet_Type); + JitOptRef set; + set = ref_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1161,8 +1157,8 @@ } case _BUILD_MAP: { - JitOptSymbol *map; - map = sym_new_type(ctx, &PyDict_Type); + JitOptRef map; + map = ref_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1192,8 +1188,8 @@ } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; - attr_st = sym_new_not_null(ctx); + JitOptRef attr_st; + attr_st = ref_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1201,10 +1197,10 @@ } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; - attr = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + JitOptRef attr; + JitOptRef self_or_null; + attr = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1213,15 +1209,15 @@ } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = sym_new_unknown(ctx); + self_or_null[0] = ref_new_unknown(ctx); } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -1230,16 +1226,16 @@ } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); - if (sym_matches_type_version(owner, type_version)) { + if (ref_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { - if (sym_set_type_version(owner, type_version)) { + if (ref_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -1257,25 +1253,25 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)offset; stack_pointer[-1] = attr; break; } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; - if (sym_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); + attr = PyJitRef_NULL; + if (ref_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; stack_pointer[-1] = attr; @@ -1284,58 +1280,58 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + attr = ref_new_const(ctx, res); } } } - if (attr == NULL) { - attr = sym_new_not_null(ctx); + if (PyJitRef_IsNull(attr)) { + attr = ref_new_not_null(ctx); } stack_pointer[-1] = attr; break; } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)hint; stack_pointer[-1] = attr; break; } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)index; stack_pointer[-1] = attr; break; } case _CHECK_ATTR_CLASS: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == sym_get_const(ctx, owner)) { + if (type == ref_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - sym_set_const(owner, type); + ref_set_const(owner, type); } } break; } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1350,7 +1346,7 @@ (void)fget; new_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)new_frame; + stack_pointer[-1].bits = (uintptr_t)new_frame; break; } @@ -1379,12 +1375,12 @@ } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef res; if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_sym_new_not_null(ctx); + res = _Py_uop_ref_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1393,8 +1389,8 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1402,16 +1398,16 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), + ref_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -1419,14 +1415,14 @@ assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); + res = ref_new_const(ctx, tmp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } else { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -1434,8 +1430,8 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1443,8 +1439,8 @@ } case _IS_OP: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1452,8 +1448,8 @@ } case _CONTAINS_OP: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1461,10 +1457,10 @@ } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PySet_Type) || - sym_matches_type(tos, &PyFrozenSet_Type)) + if (ref_matches_type(tos, &PySet_Type) || + ref_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -1472,8 +1468,8 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1481,8 +1477,8 @@ } case _CONTAINS_OP_DICT: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1490,25 +1486,25 @@ } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; - rest = sym_new_not_null(ctx); - match = sym_new_not_null(ctx); + JitOptRef rest; + JitOptRef match; + rest = ref_new_not_null(ctx); + match = ref_new_not_null(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; - b = sym_new_not_null(ctx); + JitOptRef b; + b = ref_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1516,8 +1512,8 @@ } case _IMPORT_FROM: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1529,19 +1525,19 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; - b = sym_new_not_null(ctx); + JitOptRef b; + b = ref_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - JitOptSymbol *obj; - JitOptSymbol *len; + JitOptRef obj; + JitOptRef len; obj = stack_pointer[-1]; - int tuple_length = sym_tuple_length(obj); + int tuple_length = ref_tuple_length(obj); if (tuple_length == -1) { - len = sym_new_type(ctx, &PyLong_Type); + len = ref_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1552,7 +1548,7 @@ if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = sym_new_const(ctx, temp); + len = ref_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1566,8 +1562,8 @@ } case _MATCH_CLASS: { - JitOptSymbol *attrs; - attrs = sym_new_not_null(ctx); + JitOptRef attrs; + attrs = ref_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1575,8 +1571,8 @@ } case _MATCH_MAPPING: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1584,8 +1580,8 @@ } case _MATCH_SEQUENCE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1593,8 +1589,8 @@ } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; - values_or_none = sym_new_not_null(ctx); + JitOptRef values_or_none; + values_or_none = ref_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1602,10 +1598,10 @@ } case _GET_ITER: { - JitOptSymbol *iter; - JitOptSymbol *index_or_null; - iter = sym_new_not_null(ctx); - index_or_null = sym_new_not_null(ctx); + JitOptRef iter; + JitOptRef index_or_null; + iter = ref_new_not_null(ctx); + index_or_null = ref_new_not_null(ctx); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -1614,8 +1610,8 @@ } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -1623,8 +1619,8 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1646,8 +1642,8 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1665,8 +1661,8 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1684,8 +1680,8 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; - next = sym_new_type(ctx, &PyLong_Type); + JitOptRef next; + next = ref_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1696,18 +1692,18 @@ _Py_UOpsAbstractFrame *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[0] = (JitOptSymbol *)gen_frame; + stack_pointer[0].bits = (uintptr_t)gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - method_and_self[0] = sym_new_null(ctx); + method_and_self[0] = ref_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1715,16 +1711,16 @@ } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; - method_and_self[0] = sym_new_not_null(ctx); - method_and_self[1] = sym_new_unknown(ctx); + method_and_self[0] = ref_new_not_null(ctx); + method_and_self[1] = ref_new_unknown(ctx); break; } case _WITH_EXCEPT_START: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1732,10 +1728,10 @@ } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; - prev_exc = sym_new_not_null(ctx); - new_exc = sym_new_not_null(ctx); + JitOptRef prev_exc; + JitOptRef new_exc; + prev_exc = ref_new_not_null(ctx); + new_exc = ref_new_not_null(ctx); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -1752,13 +1748,13 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1772,13 +1768,13 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1792,12 +1788,12 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1807,12 +1803,12 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1826,13 +1822,13 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1846,15 +1842,15 @@ } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1874,22 +1870,22 @@ break; } new_frame = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(ctx, callable))); + if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(ref_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); } - sym_set_type(callable, &PyFunction_Type); + ref_set_type(callable, &PyFunction_Type); break; } @@ -1910,8 +1906,8 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1919,22 +1915,22 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + ref_set_null(null); + ref_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1948,16 +1944,16 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(ctx, callable)) { - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); + assert(ref_matches_type(callable, &PyFunction_Type)); + if (ref_is_const(ctx, callable)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -1976,8 +1972,8 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; + JitOptRef *args; + JitOptRef self_or_null; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -1991,16 +1987,16 @@ } assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (ref_is_not_null(self_or_null)) { args--; argcount++; } - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2008,7 +2004,7 @@ case _PUSH_FRAME: { _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1].bits; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -2039,54 +2035,54 @@ } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); break; } case _GUARD_NOS_NOT_NULL: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_is_not_null(nos)) { + if (ref_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_non_null(nos); + ref_set_non_null(nos); break; } case _GUARD_THIRD_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-3]; - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); break; } case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyType_Type); + ref_set_const(callable, (PyObject *)&PyType_Type); break; } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + if (ref_has_type(arg)) { + res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2095,24 +2091,24 @@ } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyUnicode_Type); + ref_set_const(callable, (PyObject *)&PyUnicode_Type); break; } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_matches_type(arg, &PyUnicode_Type)) { + if (ref_matches_type(arg, &PyUnicode_Type)) { res = arg; } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2121,24 +2117,24 @@ } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyTuple_Type); + ref_set_const(callable, (PyObject *)&PyTuple_Type); break; } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_matches_type(arg, &PyTuple_Type)) { + if (ref_matches_type(arg, &PyTuple_Type)) { res = arg; } else { - res = sym_new_type(ctx, &PyTuple_Type); + res = ref_new_type(ctx, &PyTuple_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2147,17 +2143,17 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -2167,7 +2163,7 @@ _Py_UOpsAbstractFrame *init_frame; init_frame = NULL; ctx->done = true; - stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2180,8 +2176,8 @@ } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2189,8 +2185,8 @@ } case _CALL_BUILTIN_O: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2198,8 +2194,8 @@ } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2207,8 +2203,8 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2216,19 +2212,19 @@ } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (sym_get_const(ctx, callable) == len) { + if (ref_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, len); + ref_set_const(callable, len); break; } case _CALL_LEN: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyLong_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2236,31 +2232,31 @@ } case _GUARD_CALLABLE_ISINSTANCE: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (sym_get_const(ctx, callable) == isinstance) { + if (ref_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, isinstance); + ref_set_const(callable, isinstance); break; } case _CALL_ISINSTANCE: { - JitOptSymbol *cls; - JitOptSymbol *instance; - JitOptSymbol *res; + JitOptRef cls; + JitOptRef instance; + JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; - res = sym_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = sym_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); - if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + res = ref_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = ref_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); + if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { PyObject *out = Py_False; if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - sym_set_const(res, out); + ref_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } stack_pointer[-4] = res; @@ -2270,13 +2266,13 @@ } case _GUARD_CALLABLE_LIST_APPEND: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (sym_get_const(ctx, callable) == list_append) { + if (ref_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, list_append); + ref_set_const(callable, list_append); break; } @@ -2287,8 +2283,8 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2296,8 +2292,8 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2305,8 +2301,8 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2314,8 +2310,8 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2334,7 +2330,7 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2357,8 +2353,8 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2372,15 +2368,15 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; - func = sym_new_not_null(ctx); + JitOptRef func; + func = ref_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; - func_out = sym_new_not_null(ctx); + JitOptRef func_out; + func_out = ref_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2388,11 +2384,11 @@ } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); assert(corresponding_check_stack == NULL); assert(co != NULL); int framesize = co->co_framesize; @@ -2410,8 +2406,8 @@ } case _BUILD_SLICE: { - JitOptSymbol *slice; - slice = sym_new_type(ctx, &PySlice_Type); + JitOptRef slice; + slice = ref_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2419,22 +2415,22 @@ } case _CONVERT_VALUE: { - JitOptSymbol *result; - result = sym_new_not_null(ctx); + JitOptRef result; + result = ref_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2442,8 +2438,8 @@ } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2454,43 +2450,43 @@ } case _BINARY_OP: { - JitOptSymbol *rhs; - JitOptSymbol *lhs; - JitOptSymbol *res; + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - bool lhs_int = sym_matches_type(lhs, &PyLong_Type); - bool rhs_int = sym_matches_type(rhs, &PyLong_Type); - bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); - bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); + bool lhs_int = ref_matches_type(lhs, &PyLong_Type); + bool rhs_int = ref_matches_type(rhs, &PyLong_Type); + bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); + bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { if (rhs_float) { - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (lhs_float) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, rhs)) { - res = sym_new_unknown(ctx); + else if (!ref_is_const(ctx, rhs)) { + res = ref_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { - res = sym_new_type(ctx, &PyFloat_Type); + else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { + res = ref_new_type(ctx, &PyFloat_Type); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2499,11 +2495,11 @@ } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2531,61 +2527,61 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - sym_set_const(flag, Py_True); + ref_set_const(flag, Py_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - sym_set_const(flag, Py_False); + ref_set_const(flag, Py_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(val, Py_None); + ref_set_const(val, Py_None); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } stack_pointer += -1; @@ -2625,9 +2621,9 @@ } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2635,17 +2631,17 @@ } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2671,16 +2667,16 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2688,9 +2684,9 @@ } case _POP_CALL_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2698,9 +2694,9 @@ } case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2708,9 +2704,9 @@ } case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2718,10 +2714,10 @@ } case _LOAD_CONST_UNDER_INLINE: { - JitOptSymbol *value; - JitOptSymbol *new; - value = sym_new_not_null(ctx); - new = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef new; + value = ref_new_not_null(ctx); + new = ref_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -2730,10 +2726,10 @@ } case _LOAD_CONST_UNDER_INLINE_BORROW: { - JitOptSymbol *value; - JitOptSymbol *new; - value = sym_new_not_null(ctx); - new = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef new; + value = ref_new_not_null(ctx); + new = ref_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 68f1ecd1d316a4..66fa66e21935a3 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -88,6 +88,12 @@ out_of_space(JitOptContext *ctx) return &NO_SPACE_SYMBOL; } +JitOptRef +out_of_space_ref(JitOptContext *ctx) +{ + return PyJitRef_FromSymbolSteal(out_of_space(ctx)); +} + static JitOptSymbol * sym_new(JitOptContext *ctx) { @@ -98,8 +104,7 @@ sym_new(JitOptContext *ctx) return NULL; } ctx->t_arena.ty_curr_number++; - self->tag = JIT_SYM_UNKNOWN_TAG; - self->skip_refcount = DONT_SKIP_REFCOUNT; + self->tag = JIT_SYM_UNKNOWN_TAG;; return self; } @@ -107,9 +112,6 @@ static void make_const(JitOptSymbol *sym, PyObject *val) { sym->tag = JIT_SYM_KNOWN_VALUE_TAG; sym->value.value = Py_NewRef(val); - // Constants don't need to be refcounted, as they are always - // kept alive by co_consts. - sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -121,25 +123,28 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_bottom(JitOptSymbol *sym) +_Py_uop_ref_is_bottom(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(JitOptSymbol *sym) { +_Py_uop_ref_is_not_null(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return false; } @@ -150,21 +155,22 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_null(JitOptSymbol *sym) +_Py_uop_ref_is_null(JitOptRef ref) { - return sym->tag == JIT_SYM_NULL_TAG; + return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return NULL; } @@ -176,8 +182,9 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -226,11 +233,12 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) } bool -_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) +_Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_sym_set_type(ctx, sym, type); + _Py_uop_ref_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -283,8 +291,9 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int } void -_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) +_Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -305,12 +314,12 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_sym_tuple_length(sym); + Py_ssize_t len = _Py_uop_ref_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptSymbol *sym_item = _Py_uop_sym_tuple_getitem(ctx, sym, i); + JitOptRef ref_item = _Py_uop_ref_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); - _Py_uop_sym_set_const(ctx, sym_item, item); + _Py_uop_ref_set_const(ctx, ref_item, item); } make_const(sym, const_val); return; @@ -333,29 +342,29 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_sym_is_const(ctx, sym) && - _Py_uop_sym_get_const(ctx, sym) != const_val)) + (_Py_uop_ref_is_const(ctx, ref) && + _Py_uop_ref_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } - JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - PyTypeObject *type = _Py_uop_sym_get_type(value); + JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); + PyTypeObject *type = _Py_uop_ref_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: if (type == &PyBool_Type) { - _Py_uop_sym_set_const(ctx, value, Py_True); + _Py_uop_ref_set_const(ctx, value, Py_True); } } // value is falsey: else if (type == &PyBool_Type) { - _Py_uop_sym_set_const(ctx, value, Py_False); + _Py_uop_ref_set_const(ctx, value, Py_False); } else if (type == &PyLong_Type) { - _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); + _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); } else if (type == &PyUnicode_Type) { - _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); + _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); } // TODO: More types (GH-130415)! make_const(sym, const_val); @@ -364,8 +373,9 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val } void -_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -375,8 +385,9 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -385,85 +396,69 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } -void -_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - sym->skip_refcount = SKIP_REFCOUNT; -} - -void -_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - sym->skip_refcount = DONT_SKIP_REFCOUNT; -} - -bool -_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); - return sym->skip_refcount == SKIP_REFCOUNT; -} - - -JitOptSymbol * -_Py_uop_sym_new_unknown(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_new_not_null(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) +JitOptRef +_Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + JitOptRef ref = PyJitRef_FromSymbolSteal(res); + _Py_uop_ref_set_type(ctx, ref, typ); + return ref; } // Adds a new reference to const_val, owned by the symbol. -JitOptSymbol * -_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) +JitOptRef +_Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + JitOptRef ref = PyJitRef_FromSymbolSteal(res); + _Py_uop_ref_set_const(ctx, ref, const_val); + return ref; } -JitOptSymbol * -_Py_uop_sym_new_null(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); - return null_sym; + JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); + _Py_uop_ref_set_null(ctx, ref); + return ref; } PyTypeObject * -_Py_uop_sym_get_type(JitOptSymbol *sym) +_Py_uop_ref_get_type(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -486,8 +481,9 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) } unsigned int -_Py_uop_sym_get_type_version(JitOptSymbol *sym) +_Py_uop_ref_get_type_version(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -510,27 +506,28 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) } bool -_Py_uop_sym_has_type(JitOptSymbol *sym) +_Py_uop_ref_has_type(JitOptRef sym) { - return _Py_uop_sym_get_type(sym) != NULL; + return _Py_uop_ref_get_type(sym) != NULL; } bool -_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - return _Py_uop_sym_get_type(sym) == typ; + return _Py_uop_ref_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +_Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version) { - return _Py_uop_sym_get_type_version(sym) == version; + return _Py_uop_ref_get_type_version(sym) == version; } int -_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -550,7 +547,8 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, + PyJitRef_FromSymbolBorrow(value)); if (truthiness < 0) { return truthiness; } @@ -576,12 +574,12 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) return -1; } -JitOptSymbol * -_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +JitOptRef +_Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } if (size > MAX_SYMBOLIC_TUPLE_SIZE) { res->tag = JIT_SYM_KNOWN_CLASS_TAG; @@ -591,31 +589,33 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_AsSymbolBorrow(args[i]) - allocation_base(ctx)); } } - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +JitOptRef +_Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple) && item < PyTuple_GET_SIZE(tuple)) { - return _Py_uop_sym_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); + return _Py_uop_ref_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return allocation_base(ctx) + sym->tuple.items[item]; + return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); } - return _Py_uop_sym_new_not_null(ctx); + return _Py_uop_ref_new_not_null(ctx); } int -_Py_uop_sym_tuple_length(JitOptSymbol *sym) +_Py_uop_ref_tuple_length(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -644,19 +644,27 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) return false; } -JitOptSymbol * -_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +bool +_Py_uop_ref_is_immortal(JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + return _Py_uop_sym_is_immortal(sym); +} + +JitOptRef +_Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { + JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { - return value; + return ref; } JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -665,7 +673,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return res; + return PyJitRef_FromSymbolSteal(res); } // 0 on success, -1 on error. @@ -674,7 +682,7 @@ _Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -699,14 +707,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = _Py_uop_ref_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_ref_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -790,45 +798,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + JitOptRef ref = _Py_uop_ref_new_unknown(ctx); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_bottom(ref), "top is bottom"); - sym = make_bottom(ctx); - if (sym == NULL) { + ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "bottom isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -838,95 +845,86 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + _Py_uop_ref_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "42 is not immortal"); + + _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and float) isn't bottom"); + + ref = _Py_uop_ref_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "a bool is not immortal"); + + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + _Py_uop_ref_set_const(ctx, ref, val_42); + _Py_uop_ref_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and 43) isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { - goto fail; - } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_skip_refcount(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); - - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); - - JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); - JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); - JitOptSymbol *array[2] = { i1, i2 }; - sym = _Py_uop_sym_new_tuple(ctx, 2, array); + + ref = _Py_uop_ref_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_ref_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_ref_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(0) is not False"); + + JitOptRef i1 = _Py_uop_ref_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_ref_new_const(ctx, val_43); + JitOptRef array[2] = { i1, i2 }; + ref = _Py_uop_ref_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + _Py_uop_ref_matches_type(_Py_uop_ref_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - sym = _Py_uop_sym_new_const(ctx, tuple); + ref = _Py_uop_ref_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - sym = _Py_uop_sym_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_ref_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, sym, 42)), + _Py_uop_ref_is_not_null(_Py_uop_ref_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); - sym = _Py_uop_sym_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_sym_set_const(ctx, sym, Py_False); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); + JitOptRef value = _Py_uop_ref_new_type(ctx, &PyBool_Type); + ref = _Py_uop_ref_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "truthiness is not NULL"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == false, "value is constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == NULL, "value is not NULL"); + _Py_uop_ref_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == Py_False, "truthiness is not False"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == true, "value is not constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3070559db8ae57..39ffc1d7e0cbd6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -735,7 +735,7 @@ def visit(stmt: Stmt) -> None: continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_")): + if tkn.text.startswith(("sym_", "optimize_", "ref_", "PyJitRef")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index fda022a44e59cc..dd8420b792f53e 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -72,10 +72,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return "JitOptSymbol **" + return "JitOptRef *" if var.type: return var.type - return "JitOptSymbol *" + return "JitOptRef " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -124,15 +124,15 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: local.in_local = True if var.is_array(): if var.size == "1": - out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") + out.emit(f"{var.name}[0] = ref_new_not_null(ctx);\n") else: out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit(f"{var.name}[_i] = ref_new_not_null(ctx);\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = sym_new_null(ctx);\n") + out.emit(f"{var.name} = ref_new_null(ctx);\n") else: - out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + out.emit(f"{var.name} = ref_new_not_null(ctx);\n") class OptimizerEmitter(Emitter): @@ -230,7 +230,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") + stack = Stack(extract_bits=True) write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") From a1588358247f7455a8473d215bd47bd037bdcfc5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:29:46 +0800 Subject: [PATCH 09/24] refactor more --- Include/internal/pycore_optimizer.h | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 78cea6198d9767..034b93525f9b43 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -183,49 +183,37 @@ typedef enum _JitSymType { typedef struct _jit_opt_known_class { - struct { - uint8_t tag; - }; + uint8_t tag; uint32_t version; PyTypeObject *type; } JitOptKnownClass; typedef struct _jit_opt_known_version { - struct { - uint8_t tag; - }; + uint8_t tag; uint32_t version; } JitOptKnownVersion; typedef struct _jit_opt_known_value { - struct { - uint8_t tag; - }; + uint8_t tag; PyObject *value; } JitOptKnownValue; #define MAX_SYMBOLIC_TUPLE_SIZE 7 typedef struct _jit_opt_tuple { - struct { - uint8_t tag; - }; + uint8_t tag; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; } JitOptTuple; typedef struct { - struct { - uint8_t tag; - }; + uint8_t tag; bool invert; uint16_t value; } JitOptTruthiness; typedef union _jit_opt_symbol { - struct { - uint8_t tag; - }; + uint8_t tag; JitOptKnownClass cls; JitOptKnownValue value; JitOptKnownVersion version; From e77f842bcbd8ef3a387a92ec88d22626b156b9a1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:31:33 +0800 Subject: [PATCH 10/24] fix debug build --- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 2 +- Python/optimizer_symbols.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 333971a2e02828..3b3c08a81e0bbf 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -750,7 +750,7 @@ dummy_func(void) { } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (ref_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f59a9cbd6d217a..d70bf1ee22f55a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1985,7 +1985,7 @@ ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (ref_is_not_null(self_or_null)) { args--; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 66fa66e21935a3..d0a296a83e1e59 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -745,7 +745,7 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = PyJitRef_NULL; } #endif From 01004c2404941e3c2dcdde3f11870c2cc175cef2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:33:08 +0800 Subject: [PATCH 11/24] lint --- Python/optimizer_symbols.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index d0a296a83e1e59..994f591c561d6c 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -396,7 +396,7 @@ _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) } } -JitOptRef +JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); @@ -406,7 +406,7 @@ _Py_uop_ref_new_unknown(JitOptContext *ctx) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); @@ -417,7 +417,7 @@ _Py_uop_ref_new_not_null(JitOptContext *ctx) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); @@ -430,7 +430,7 @@ _Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) } // Adds a new reference to const_val, owned by the symbol. -JitOptRef +JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); @@ -443,7 +443,7 @@ _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) return ref; } -JitOptRef +JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); @@ -574,7 +574,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) return -1; } -JitOptRef +JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); @@ -595,7 +595,7 @@ _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); @@ -651,7 +651,7 @@ _Py_uop_ref_is_immortal(JitOptRef ref) return _Py_uop_sym_is_immortal(sym); } -JitOptRef +JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); From 0189413bba10f34cc288b59b7c728bd6485b7f1d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 00:50:54 +0800 Subject: [PATCH 12/24] fix upstream --- Python/optimizer_bytecodes.c | 4 ++-- Python/optimizer_cases.c.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d0183a0bec4fe4..1af6aafc0d1297 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -927,10 +927,10 @@ dummy_func(void) { } op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - if (sym_matches_type(iter, &PyTuple_Type)) { + if (ref_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(iter, &PyTuple_Type); + ref_set_type(iter, &PyTuple_Type); } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index df5a2715121ff3..c8f37aa0b88dae 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1651,12 +1651,12 @@ } case _ITER_CHECK_TUPLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = stack_pointer[-2]; - if (sym_matches_type(iter, &PyTuple_Type)) { + if (ref_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(iter, &PyTuple_Type); + ref_set_type(iter, &PyTuple_Type); break; } From 4a386bf79173f41fb3f90c105e84478237807408 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 00:52:38 +0800 Subject: [PATCH 13/24] reduce diff --- Include/internal/pycore_optimizer.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 034b93525f9b43..72230e7970d0ae 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -181,7 +181,6 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; - typedef struct _jit_opt_known_class { uint8_t tag; uint32_t version; From ac034a004578ff23f9fdfed11c2cc4c7c57eb19f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 01:14:32 +0800 Subject: [PATCH 14/24] fix for FT --- Include/internal/pycore_optimizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 72230e7970d0ae..47efc3ce2507d8 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -258,7 +258,7 @@ PyJitRef_Borrow(JitOptRef ref) return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; } -static const JitOptRef PyJitRef_NULL = { .bits = PyStackRef_NULL_BITS }; +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL.bits}; static inline bool PyJitRef_IsNull(JitOptRef ref) From b6e467e79323883ad7904f1c5fdf009e1eb6b1c2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 02:48:07 +0800 Subject: [PATCH 15/24] fix failing tests --- Lib/test/test_generated_cases.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 37046d8e1c02b7..aaed3fba13e758 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2002,8 +2002,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - JitOptSymbol *arg1; - JitOptSymbol *out; + JitOptRef arg1; + JitOptRef out; arg1 = stack_pointer[-1]; out = EGGS(arg1); stack_pointer[-1] = out; @@ -2011,8 +2011,8 @@ def test_overridden_abstract_args(self): } case OP2: { - JitOptSymbol *out; - out = sym_new_not_null(ctx); + JitOptRef out; + out = ref_new_not_null(ctx); stack_pointer[-1] = out; break; } @@ -2036,14 +2036,14 @@ def test_no_overridden_case(self): """ output = """ case OP: { - JitOptSymbol *out; - out = sym_new_not_null(ctx); + JitOptRef out; + out = ref_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = NULL; stack_pointer[-1] = out; break; @@ -2177,7 +2177,7 @@ def test_validate_uop_unused_output(self): """ output = """ case OP: { - JitOptSymbol *foo; + JitOptRef foo; foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; From 3a3fa9d3138cff29d3ad405b4cecd314989b3a21 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:01:50 +0800 Subject: [PATCH 16/24] Fix for disabled GIL --- Include/internal/pycore_optimizer.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 47efc3ce2507d8..d4e65b72212bd8 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -258,7 +258,11 @@ PyJitRef_Borrow(JitOptRef ref) return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; } -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL.bits}; +#ifndef Py_GIL_DISABLED +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL_BITS}; +#else +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; +#endif static inline bool PyJitRef_IsNull(JitOptRef ref) From ab1ad9c62eb575d8dd0080a3daae48bff48dc23b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:08:07 +0800 Subject: [PATCH 17/24] fix on FT again --- Include/internal/pycore_optimizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d4e65b72212bd8..633ffbb3a1f2c0 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -267,7 +267,7 @@ static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; static inline bool PyJitRef_IsNull(JitOptRef ref) { - return ref.bits == PyStackRef_NULL_BITS; + return ref.bits == PyJitRef_NULL.bits; } static inline int From 5d82489bddf113d6f3583a8154cffa9ec46f6e09 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:54:49 +0800 Subject: [PATCH 18/24] Try fix windows --- Include/internal/pycore_optimizer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 633ffbb3a1f2c0..49f71dc0734128 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -259,9 +259,9 @@ PyJitRef_Borrow(JitOptRef ref) } #ifndef Py_GIL_DISABLED -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL_BITS}; +static const JitOptRef PyJitRef_NULL = {.bits = PyStackRef_NULL_BITS}; #else -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; +static const JitOptRef PyJitRef_NULL = {.bits = Py_TAG_DEFERRED}; #endif static inline bool From 4d9a68e257ecd8c3b7637940fd8dc1f301e1a085 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 22:22:00 +0800 Subject: [PATCH 19/24] Apply code review suggestions from Tomas --- Include/internal/pycore_optimizer.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 49f71dc0734128..3d89fea9ce389a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -238,18 +238,18 @@ PyJitRef_AsSymbolBorrow(JitOptRef ref) bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); static inline JitOptRef -PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; - } - return (JitOptRef){.bits=(uintptr_t)sym}; + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; } static inline JitOptRef -PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) +PyJitRef_FromSymbolSteal(JitOptSymbol *sym) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + return PyJitRef_FromSymbolBorrow(sym); + } + return (JitOptRef){.bits=(uintptr_t)sym}; } static inline JitOptRef From 2bbd47a9514d701eff96637720bffa1e4dfd16f7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 23:07:08 +0800 Subject: [PATCH 20/24] call the functions sym instead of ref --- Include/internal/pycore_optimizer.h | 52 +- Lib/test/test_generated_cases.py | 4 +- Python/optimizer_analysis.c | 60 +- Python/optimizer_bytecodes.c | 548 ++++++++-------- Python/optimizer_cases.c.h | 642 +++++++++---------- Python/optimizer_symbols.c | 258 ++++---- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/optimizer_generator.py | 8 +- 8 files changed, 787 insertions(+), 787 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 3d89fea9ce389a..88acea076dbc80 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -235,7 +235,7 @@ PyJitRef_AsSymbolBorrow(JitOptRef ref) return JIT_BITS_TO_PTR_MASKED(ref); } -bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); +bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); static inline JitOptRef PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) @@ -246,7 +246,7 @@ PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) static inline JitOptRef PyJitRef_FromSymbolSteal(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { return PyJitRef_FromSymbolBorrow(sym); } return (JitOptRef){.bits=(uintptr_t)sym}; @@ -311,31 +311,31 @@ typedef struct _JitOptContext { JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_ref_is_null(JitOptRef sym); -extern bool _Py_uop_ref_is_not_null(JitOptRef sym); -extern bool _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef sym); -extern PyObject *_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx); -extern JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx); -extern JitOptRef _Py_uop_ref_new_type( +extern bool _Py_uop_sym_is_null(JitOptRef sym); +extern bool _Py_uop_sym_is_not_null(JitOptRef sym); +extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx); -extern bool _Py_uop_ref_has_type(JitOptRef sym); -extern bool _Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ); -extern bool _Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version); -extern void _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef sym); -extern void _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef sym); -extern void _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); -extern bool _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); -extern void _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); -extern bool _Py_uop_ref_is_bottom(JitOptRef sym); -extern int _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef sym); -extern PyTypeObject *_Py_uop_ref_get_type(JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); -extern JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); -extern int _Py_uop_ref_tuple_length(JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); +extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); +extern bool _Py_uop_sym_has_type(JitOptRef sym); +extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(JitOptRef sym); +extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); +extern int _Py_uop_sym_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index aaed3fba13e758..b609c2bb297078 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2012,7 +2012,7 @@ def test_overridden_abstract_args(self): case OP2: { JitOptRef out; - out = ref_new_not_null(ctx); + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } @@ -2037,7 +2037,7 @@ def test_no_overridden_case(self): output = """ case OP: { JitOptRef out; - out = ref_new_not_null(ctx); + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 874d4482f24f5f..de337f637163ba 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -315,33 +315,33 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ -#define ref_is_not_null _Py_uop_ref_is_not_null -#define ref_is_const _Py_uop_ref_is_const -#define ref_get_const _Py_uop_ref_get_const -#define ref_new_unknown _Py_uop_ref_new_unknown -#define ref_new_not_null _Py_uop_ref_new_not_null -#define ref_new_type _Py_uop_ref_new_type -#define ref_is_null _Py_uop_ref_is_null -#define ref_new_const _Py_uop_ref_new_const -#define ref_new_null _Py_uop_ref_new_null -#define ref_has_type _Py_uop_ref_has_type -#define ref_get_type _Py_uop_ref_get_type -#define ref_matches_type _Py_uop_ref_matches_type -#define ref_matches_type_version _Py_uop_ref_matches_type_version -#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) -#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) -#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) -#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) -#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) -#define ref_is_bottom _Py_uop_ref_is_bottom -#define ref_truthiness _Py_uop_ref_truthiness +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_has_type _Py_uop_sym_has_type +#define sym_get_type _Py_uop_sym_get_type +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define ref_new_tuple _Py_uop_ref_new_tuple -#define ref_tuple_getitem _Py_uop_ref_tuple_getitem -#define ref_tuple_length _Py_uop_ref_tuple_length -#define ref_is_immortal _Py_uop_ref_is_immortal -#define ref_new_truthiness _Py_uop_ref_new_truthiness +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness static int optimize_to_bool( @@ -350,16 +350,16 @@ optimize_to_bool( JitOptRef value, JitOptRef *result_ptr) { - if (ref_matches_type(value, &PyBool_Type)) { + if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); *result_ptr = value; return 1; } - int truthiness = ref_truthiness(ctx, value); + int truthiness = sym_truthiness(ctx, value); if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - *result_ptr = ref_new_const(ctx, load); + *result_ptr = sym_new_const(ctx, load); return 1; } return 0; @@ -386,10 +386,10 @@ lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); - return ref_new_const(ctx, lookup); + return sym_new_const(ctx, lookup); } } - return ref_new_not_null(ctx); + return sym_new_not_null(ctx); } /* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 1af6aafc0d1297..7d65a9187b2497 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -9,32 +9,32 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; /* Shortened forms for convenience */ -#define ref_is_not_null _Py_uop_ref_is_not_null -#define ref_is_const _Py_uop_ref_is_const -#define ref_get_const _Py_uop_ref_get_const -#define ref_new_unknown _Py_uop_ref_new_unknown -#define ref_new_not_null _Py_uop_ref_new_not_null -#define ref_new_type _Py_uop_ref_new_type -#define ref_is_null _Py_uop_ref_is_null -#define ref_new_const _Py_uop_ref_new_const -#define ref_new_null _Py_uop_ref_new_null -#define ref_matches_type _Py_uop_ref_matches_type -#define ref_matches_type_version _Py_uop_ref_matches_type_version -#define ref_get_type _Py_uop_ref_get_type -#define ref_has_type _Py_uop_ref_has_type -#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) -#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) -#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) -#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) -#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) -#define ref_is_bottom _Py_uop_ref_is_bottom +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define ref_new_tuple _Py_uop_ref_new_tuple -#define ref_tuple_getitem _Py_uop_ref_tuple_getitem -#define ref_tuple_length _Py_uop_ref_tuple_length -#define ref_is_immortal _Py_uop_ref_is_immortal -#define ref_new_truthiness _Py_uop_ref_new_truthiness +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness extern int optimize_to_bool( @@ -77,7 +77,7 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (ref_is_null(value)) { + if (sym_is_null(value)) { ctx->done = true; } } @@ -92,7 +92,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptRef temp = ref_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -101,38 +101,38 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = ref_new_null(ctx); + res = sym_new_null(ctx); } op(_GUARD_TOS_INT, (value -- value)) { - if (ref_matches_type(value, &PyLong_Type)) { + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyLong_Type); + sym_set_type(value, &PyLong_Type); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (ref_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyLong_Type); + sym_set_type(left, &PyLong_Type); } op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == ref_get_const(ctx, owner)) { + if (type == sym_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - ref_set_const(owner, type); + sym_set_const(owner, type); } } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); - if (ref_matches_type_version(owner, type_version)) { + if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this @@ -144,7 +144,7 @@ dummy_func(void) { // if it wasn't this means that the type version was previously set to something else // and we set the owner to bottom, so we don't need to add a watcher because we must have // already added one earlier. - if (ref_set_type_version(owner, type_version)) { + if (sym_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -154,27 +154,27 @@ dummy_func(void) { } op(_GUARD_TOS_FLOAT, (value -- value)) { - if (ref_matches_type(value, &PyFloat_Type)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - if (ref_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); } op(_BINARY_OP, (lhs, rhs -- res)) { - bool lhs_int = ref_matches_type(lhs, &PyLong_Type); - bool rhs_int = ref_matches_type(rhs, &PyLong_Type); - bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); - bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { // This one's fun... the *type* of the result depends on the @@ -191,110 +191,110 @@ dummy_func(void) { if (rhs_float) { // Case D, E, F, or G... can't know without the sign of the LHS // or whether the RHS is whole, which isn't worth the effort: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (lhs_float) { // Case C: - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } - else if (!ref_is_const(ctx, rhs)) { + else if (!sym_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { // Case B: - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else { // Case A: - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -303,22 +303,22 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -327,22 +327,22 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -351,35 +351,35 @@ dummy_func(void) { } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { JitOptRef res; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; @@ -391,109 +391,109 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { - assert(ref_matches_type(tuple_st, &PyTuple_Type)); - if (ref_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); - long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = ref_tuple_length(tuple_st); + int tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } else { assert(index < tuple_length); - res = ref_tuple_getitem(ctx, tuple_st, index); + res = sym_tuple_getitem(ctx, tuple_st, index); } } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } } op(_TO_BOOL, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - ref_set_type(value, &PyBool_Type); - value = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyBool_Type); + value = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_INT, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_type(value, &PyLong_Type); - res = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyLong_Type); + res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_LIST, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_const(value, Py_None); - res = ref_new_const(ctx, Py_False); + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } } op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyUnicode_Type)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyUnicode_Type); + sym_set_type(nos, &PyUnicode_Type); } op(_GUARD_TOS_UNICODE, (value -- value)) { - if (ref_matches_type(value, &PyUnicode_Type)) { + if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyUnicode_Type); + sym_set_type(value, &PyUnicode_Type); } op(_TO_BOOL_STR, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } } op(_UNARY_NOT, (value -- res)) { - ref_set_type(value, &PyBool_Type); - res = ref_new_truthiness(ctx, value, false); + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); } op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_ref_new_not_null(ctx); + res = _Py_uop_sym_new_not_null(ctx); } } op(_COMPARE_OP_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), - ref_get_const(ctx, right), + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), + sym_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -501,42 +501,42 @@ dummy_func(void) { assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = ref_new_const(ctx, tmp); + res = sym_new_const(ctx, tmp); Py_DECREF(tmp); } else { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_COMPARE_OP_FLOAT, (left, right -- res)) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_SET, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_DICT, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -544,35 +544,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -588,7 +588,7 @@ dummy_func(void) { } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)offset; } @@ -596,8 +596,8 @@ dummy_func(void) { (void)dict_version; (void)index; attr = PyJitRef_NULL; - if (ref_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); @@ -605,20 +605,20 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = ref_new_const(ctx, res); + attr = sym_new_const(ctx, res); } } } if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); } } op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = ref_new_null(ctx); + null[0] = sym_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -627,25 +627,25 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = ref_new_unknown(ctx); + self_or_null[0] = sym_new_unknown(ctx); } } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)hint; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)index; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -654,7 +654,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -663,7 +663,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -672,7 +672,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -682,7 +682,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -692,7 +692,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -707,26 +707,26 @@ dummy_func(void) { } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(ref_get_const(ctx, callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } - ref_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(ref_matches_type(callable, &PyFunction_Type)); - if (ref_is_const(ctx, callable)) { - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(ctx, callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -734,8 +734,8 @@ dummy_func(void) { } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - ref_set_null(null); - ref_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -752,13 +752,13 @@ dummy_func(void) { assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); - if (ref_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -768,8 +768,8 @@ dummy_func(void) { op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -792,8 +792,8 @@ dummy_func(void) { op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { @@ -831,7 +831,7 @@ dummy_func(void) { ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -849,7 +849,7 @@ dummy_func(void) { } op(_YIELD_VALUE, (unused -- value)) { - value = ref_new_unknown(ctx); + value = sym_new_unknown(ctx); } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { @@ -913,7 +913,7 @@ dummy_func(void) { (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } } @@ -922,47 +922,47 @@ dummy_func(void) { /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } } op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - if (ref_matches_type(iter, &PyTuple_Type)) { + if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(iter, &PyTuple_Type); + sym_set_type(iter, &PyTuple_Type); } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { - next = ref_new_type(ctx, &PyLong_Type); + next = sym_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (ref_has_type(arg)) { - res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); + if (sym_has_type(arg)) { + res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } } op(_CALL_STR_1, (unused, unused, arg -- res)) { - if (ref_matches_type(arg, &PyUnicode_Type)) { + if (sym_matches_type(arg, &PyUnicode_Type)) { // e.g. str('foo') or str(foo) where foo is known to be a string res = arg; } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } } op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { // the result is always a bool, but sometimes we can // narrow it down to True or False - res = ref_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = ref_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); - if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { // isinstance(inst, cls) where both inst and cls have // known types, meaning we can deduce either True or False @@ -971,50 +971,50 @@ dummy_func(void) { if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - ref_set_const(res, out); + sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } op(_GUARD_IS_TRUE_POP, (flag -- )) { - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - ref_set_const(flag, Py_True); + sym_set_const(flag, Py_True); } op(_GUARD_IS_FALSE_POP, (flag -- )) { - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - ref_set_const(flag, Py_False); + sym_set_const(flag, Py_False); } op(_GUARD_IS_NONE_POP, (val -- )) { - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - ref_set_const(val, Py_None); + sym_set_const(val, Py_None); } op(_GUARD_IS_NOT_NONE_POP, (val -- )) { - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -1028,13 +1028,13 @@ dummy_func(void) { } op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[0] = ref_new_null(ctx); + method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; } op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { - method_and_self[0] = ref_new_not_null(ctx); - method_and_self[1] = ref_new_unknown(ctx); + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { @@ -1048,154 +1048,154 @@ dummy_func(void) { op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = ref_new_const(ctx, Py_True); + res = sym_new_const(ctx, Py_True); } op(_BUILD_TUPLE, (values[oparg] -- tup)) { - tup = ref_new_tuple(ctx, oparg, values); + tup = sym_new_tuple(ctx, oparg, values); } op(_BUILD_LIST, (values[oparg] -- list)) { - list = ref_new_type(ctx, &PyList_Type); + list = sym_new_type(ctx, &PyList_Type); } op(_BUILD_SLICE, (args[oparg] -- slice)) { - slice = ref_new_type(ctx, &PySlice_Type); + slice = sym_new_type(ctx, &PySlice_Type); } op(_BUILD_MAP, (values[oparg*2] -- map)) { - map = ref_new_type(ctx, &PyDict_Type); + map = sym_new_type(ctx, &PyDict_Type); } op(_BUILD_STRING, (pieces[oparg] -- str)) { - str = ref_new_type(ctx, &PyUnicode_Type); + str = sym_new_type(ctx, &PyUnicode_Type); } op(_BUILD_SET, (values[oparg] -- set)) { - set = ref_new_type(ctx, &PySet_Type); + set = sym_new_type(ctx, &PySet_Type); } op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { - val0 = ref_tuple_getitem(ctx, seq, 0); - val1 = ref_tuple_getitem(ctx, seq, 1); + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); } op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { for (int i = 0; i < oparg; i++) { - values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } } op(_CALL_TUPLE_1, (callable, null, arg -- res)) { - if (ref_matches_type(arg, &PyTuple_Type)) { + if (sym_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple res = arg; } else { - res = ref_new_type(ctx, &PyTuple_Type); + res = sym_new_type(ctx, &PyTuple_Type); } } op(_GUARD_TOS_LIST, (tos -- tos)) { - if (ref_matches_type(tos, &PyList_Type)) { + if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyList_Type); + sym_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyList_Type)) { + if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyList_Type); + sym_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { - if (ref_matches_type(tos, &PyTuple_Type)) { + if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyTuple_Type); + sym_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyTuple_Type)) { + if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyTuple_Type); + sym_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { - if (ref_matches_type(tos, &PyDict_Type)) { + if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyDict_Type); + sym_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyDict_Type)) { + if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyDict_Type); + sym_set_type(nos, &PyDict_Type); } op(_GUARD_TOS_ANY_SET, (tos -- tos)) { - if (ref_matches_type(tos, &PySet_Type) || - ref_matches_type(tos, &PyFrozenSet_Type)) + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - if (ref_is_not_null(nos)) { + if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_non_null(nos); + sym_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyType_Type); + sym_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyTuple_Type); + sym_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyUnicode_Type); + sym_set_const(callable, (PyObject *)&PyUnicode_Type); } op(_CALL_LEN, (unused, unused, unused -- res)) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } op(_GET_LEN, (obj -- obj, len)) { - int tuple_length = ref_tuple_length(obj); + int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { - len = ref_new_type(ctx, &PyLong_Type); + len = sym_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1206,33 +1206,33 @@ dummy_func(void) { if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = ref_new_const(ctx, temp); + len = sym_new_const(ctx, temp); Py_DECREF(temp); } } op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (ref_get_const(ctx, callable) == len) { + if (sym_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, len); + sym_set_const(callable, len); } op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (ref_get_const(ctx, callable) == isinstance) { + if (sym_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, isinstance); + sym_set_const(callable, isinstance); } op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (ref_get_const(ctx, callable) == list_append) { + if (sym_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, list_append); + sym_set_const(callable, list_append); } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c8f37aa0b88dae..68caa24947755e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -28,7 +28,7 @@ case _LOAD_FAST_CHECK: { JitOptRef value; value = GETLOCAL(oparg); - if (ref_is_null(value)) { + if (sym_is_null(value)) { ctx->done = true; } stack_pointer[0] = value; @@ -58,7 +58,7 @@ case _LOAD_FAST_AND_CLEAR: { JitOptRef value; value = GETLOCAL(oparg); - JitOptRef temp = ref_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -70,7 +70,7 @@ JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -83,7 +83,7 @@ assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -113,7 +113,7 @@ case _PUSH_NULL: { JitOptRef res; - res = ref_new_null(ctx); + res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -134,7 +134,7 @@ case _END_SEND: { JitOptRef val; - val = ref_new_not_null(ctx); + val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -143,7 +143,7 @@ case _UNARY_NEGATIVE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -152,8 +152,8 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - ref_set_type(value, &PyBool_Type); - res = ref_new_truthiness(ctx, value, false); + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); stack_pointer[-1] = res; break; } @@ -164,7 +164,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -175,8 +175,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - ref_set_type(value, &PyBool_Type); - value = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyBool_Type); + value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; @@ -188,8 +188,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_type(value, &PyLong_Type); - res = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyLong_Type); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -198,20 +198,20 @@ case _GUARD_NOS_LIST: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyList_Type)) { + if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyList_Type); + sym_set_type(nos, &PyList_Type); break; } case _GUARD_TOS_LIST: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyList_Type)) { + if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyList_Type); + sym_set_type(tos, &PyList_Type); break; } @@ -225,7 +225,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; @@ -237,8 +237,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_const(value, Py_None); - res = ref_new_const(ctx, Py_False); + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } stack_pointer[-1] = res; break; @@ -247,20 +247,20 @@ case _GUARD_NOS_UNICODE: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyUnicode_Type)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyUnicode_Type); + sym_set_type(nos, &PyUnicode_Type); break; } case _GUARD_TOS_UNICODE: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyUnicode_Type)) { + if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyUnicode_Type); + sym_set_type(value, &PyUnicode_Type); break; } @@ -270,7 +270,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -279,14 +279,14 @@ case _REPLACE_WITH_TRUE: { JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = ref_new_const(ctx, Py_True); + res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -294,20 +294,20 @@ case _GUARD_NOS_INT: { JitOptRef left; left = stack_pointer[-2]; - if (ref_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyLong_Type); + sym_set_type(left, &PyLong_Type); break; } case _GUARD_TOS_INT: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyLong_Type)) { + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyLong_Type); + sym_set_type(value, &PyLong_Type); break; } @@ -317,22 +317,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -345,22 +345,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -373,22 +373,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -398,20 +398,20 @@ case _GUARD_NOS_FLOAT: { JitOptRef left; left = stack_pointer[-2]; - if (ref_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyFloat_Type)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); break; } @@ -421,23 +421,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -453,23 +453,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -485,23 +485,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -513,7 +513,7 @@ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -522,7 +522,7 @@ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -531,7 +531,7 @@ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -544,21 +544,21 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -571,18 +571,18 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; JitOptRef res; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; @@ -596,7 +596,7 @@ case _BINARY_OP_EXTEND: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -605,7 +605,7 @@ case _BINARY_SLICE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -620,7 +620,7 @@ case _BINARY_OP_SUBSCR_LIST_INT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -629,7 +629,7 @@ case _BINARY_OP_SUBSCR_LIST_SLICE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -638,7 +638,7 @@ case _BINARY_OP_SUBSCR_STR_INT: { JitOptRef res; - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -648,20 +648,20 @@ case _GUARD_NOS_TUPLE: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyTuple_Type)) { + if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyTuple_Type); + sym_set_type(nos, &PyTuple_Type); break; } case _GUARD_TOS_TUPLE: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyTuple_Type)) { + if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyTuple_Type); + sym_set_type(tos, &PyTuple_Type); break; } @@ -671,22 +671,22 @@ JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; - assert(ref_matches_type(tuple_st, &PyTuple_Type)); - if (ref_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); - long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = ref_tuple_length(tuple_st); + int tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } else { assert(index < tuple_length); - res = ref_tuple_getitem(ctx, tuple_st, index); + res = sym_tuple_getitem(ctx, tuple_st, index); } } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -697,26 +697,26 @@ case _GUARD_NOS_DICT: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyDict_Type)) { + if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyDict_Type); + sym_set_type(nos, &PyDict_Type); break; } case _GUARD_TOS_DICT: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyDict_Type)) { + if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyDict_Type); + sym_set_type(tos, &PyDict_Type); break; } case _BINARY_OP_SUBSCR_DICT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -725,7 +725,7 @@ case _BINARY_OP_SUBSCR_CHECK_FUNC: { JitOptRef getitem; - getitem = ref_new_not_null(ctx); + getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -780,14 +780,14 @@ case _CALL_INTRINSIC_1: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -823,14 +823,14 @@ case _GET_AITER: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { JitOptRef awaitable; - awaitable = ref_new_not_null(ctx); + awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -839,7 +839,7 @@ case _GET_AWAITABLE: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -856,7 +856,7 @@ case _YIELD_VALUE: { JitOptRef value; - value = ref_new_unknown(ctx); + value = sym_new_unknown(ctx); stack_pointer[-1] = value; break; } @@ -869,7 +869,7 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -878,7 +878,7 @@ case _LOAD_BUILD_CLASS: { JitOptRef bc; - bc = ref_new_not_null(ctx); + bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -902,7 +902,7 @@ top = &stack_pointer[-1 + oparg]; (void)top; for (int i = 0; i < oparg; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -914,8 +914,8 @@ JitOptRef val1; JitOptRef val0; seq = stack_pointer[-1]; - val0 = ref_tuple_getitem(ctx, seq, 0); - val1 = ref_tuple_getitem(ctx, seq, 1); + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -929,7 +929,7 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { - values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -940,7 +940,7 @@ JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = ref_new_not_null(ctx); + values[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -955,7 +955,7 @@ (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -986,7 +986,7 @@ case _LOAD_LOCALS: { JitOptRef locals; - locals = ref_new_not_null(ctx); + locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -997,7 +997,7 @@ case _LOAD_NAME: { JitOptRef v; - v = ref_new_not_null(ctx); + v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1007,7 +1007,7 @@ case _LOAD_GLOBAL: { JitOptRef *res; res = &stack_pointer[0]; - res[0] = ref_new_not_null(ctx); + res[0] = sym_new_not_null(ctx); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1018,7 +1018,7 @@ null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = ref_new_null(ctx); + null[0] = sym_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1034,7 +1034,7 @@ case _LOAD_GLOBAL_MODULE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1043,7 +1043,7 @@ case _LOAD_GLOBAL_BUILTINS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1064,14 +1064,14 @@ case _LOAD_FROM_DICT_OR_DEREF: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1090,7 +1090,7 @@ case _BUILD_STRING: { JitOptRef str; - str = ref_new_type(ctx, &PyUnicode_Type); + str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1099,7 +1099,7 @@ case _BUILD_INTERPOLATION: { JitOptRef interpolation; - interpolation = ref_new_not_null(ctx); + interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1108,7 +1108,7 @@ case _BUILD_TEMPLATE: { JitOptRef template; - template = ref_new_not_null(ctx); + template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1119,7 +1119,7 @@ JitOptRef *values; JitOptRef tup; values = &stack_pointer[-oparg]; - tup = ref_new_tuple(ctx, oparg, values); + tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1128,7 +1128,7 @@ case _BUILD_LIST: { JitOptRef list; - list = ref_new_type(ctx, &PyList_Type); + list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1149,7 +1149,7 @@ case _BUILD_SET: { JitOptRef set; - set = ref_new_type(ctx, &PySet_Type); + set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1158,7 +1158,7 @@ case _BUILD_MAP: { JitOptRef map; - map = ref_new_type(ctx, &PyDict_Type); + map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1189,7 +1189,7 @@ case _LOAD_SUPER_ATTR_ATTR: { JitOptRef attr_st; - attr_st = ref_new_not_null(ctx); + attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1199,8 +1199,8 @@ case _LOAD_SUPER_ATTR_METHOD: { JitOptRef attr; JitOptRef self_or_null; - attr = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1215,9 +1215,9 @@ owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = ref_new_unknown(ctx); + self_or_null[0] = sym_new_unknown(ctx); } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -1230,12 +1230,12 @@ owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); - if (ref_matches_type_version(owner, type_version)) { + if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { - if (ref_set_type_version(owner, type_version)) { + if (sym_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -1255,7 +1255,7 @@ case _LOAD_ATTR_INSTANCE_VALUE: { JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)offset; stack_pointer[-1] = attr; break; @@ -1270,8 +1270,8 @@ (void)dict_version; (void)index; attr = PyJitRef_NULL; - if (ref_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; stack_pointer[-1] = attr; @@ -1280,12 +1280,12 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = ref_new_const(ctx, res); + attr = sym_new_const(ctx, res); } } } if (PyJitRef_IsNull(attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; break; @@ -1294,7 +1294,7 @@ case _LOAD_ATTR_WITH_HINT: { JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)hint; stack_pointer[-1] = attr; break; @@ -1303,7 +1303,7 @@ case _LOAD_ATTR_SLOT: { JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)index; stack_pointer[-1] = attr; break; @@ -1315,11 +1315,11 @@ uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == ref_get_const(ctx, owner)) { + if (type == sym_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - ref_set_const(owner, type); + sym_set_const(owner, type); } } break; @@ -1331,7 +1331,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1377,10 +1377,10 @@ case _COMPARE_OP: { JitOptRef res; if (oparg & 16) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_ref_new_not_null(ctx); + res = _Py_uop_sym_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1390,7 +1390,7 @@ case _COMPARE_OP_FLOAT: { JitOptRef res; - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1403,11 +1403,11 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), - ref_get_const(ctx, right), + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), + sym_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -1415,14 +1415,14 @@ assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = ref_new_const(ctx, tmp); + res = sym_new_const(ctx, tmp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } else { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -1431,7 +1431,7 @@ case _COMPARE_OP_STR: { JitOptRef res; - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1440,7 +1440,7 @@ case _IS_OP: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1449,7 +1449,7 @@ case _CONTAINS_OP: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1459,8 +1459,8 @@ case _GUARD_TOS_ANY_SET: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PySet_Type) || - ref_matches_type(tos, &PyFrozenSet_Type)) + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -1469,7 +1469,7 @@ case _CONTAINS_OP_SET: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1478,7 +1478,7 @@ case _CONTAINS_OP_DICT: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1488,8 +1488,8 @@ case _CHECK_EG_MATCH: { JitOptRef rest; JitOptRef match; - rest = ref_new_not_null(ctx); - match = ref_new_not_null(ctx); + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; @@ -1497,14 +1497,14 @@ case _CHECK_EXC_MATCH: { JitOptRef b; - b = ref_new_not_null(ctx); + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1513,7 +1513,7 @@ case _IMPORT_FROM: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1526,7 +1526,7 @@ case _IS_NONE: { JitOptRef b; - b = ref_new_not_null(ctx); + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } @@ -1535,9 +1535,9 @@ JitOptRef obj; JitOptRef len; obj = stack_pointer[-1]; - int tuple_length = ref_tuple_length(obj); + int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { - len = ref_new_type(ctx, &PyLong_Type); + len = sym_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1548,7 +1548,7 @@ if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = ref_new_const(ctx, temp); + len = sym_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1563,7 +1563,7 @@ case _MATCH_CLASS: { JitOptRef attrs; - attrs = ref_new_not_null(ctx); + attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1572,7 +1572,7 @@ case _MATCH_MAPPING: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1581,7 +1581,7 @@ case _MATCH_SEQUENCE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1590,7 +1590,7 @@ case _MATCH_KEYS: { JitOptRef values_or_none; - values_or_none = ref_new_not_null(ctx); + values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1600,8 +1600,8 @@ case _GET_ITER: { JitOptRef iter; JitOptRef index_or_null; - iter = ref_new_not_null(ctx); - index_or_null = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); + index_or_null = sym_new_not_null(ctx); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -1611,7 +1611,7 @@ case _GET_YIELD_FROM_ITER: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -1620,7 +1620,7 @@ case _FOR_ITER_TIER_TWO: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1643,7 +1643,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1653,10 +1653,10 @@ case _ITER_CHECK_TUPLE: { JitOptRef iter; iter = stack_pointer[-2]; - if (ref_matches_type(iter, &PyTuple_Type)) { + if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(iter, &PyTuple_Type); + sym_set_type(iter, &PyTuple_Type); break; } @@ -1668,7 +1668,7 @@ case _ITER_NEXT_TUPLE: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1687,7 +1687,7 @@ case _ITER_NEXT_RANGE: { JitOptRef next; - next = ref_new_type(ctx, &PyLong_Type); + next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1709,7 +1709,7 @@ JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - method_and_self[0] = ref_new_null(ctx); + method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1719,14 +1719,14 @@ case _LOAD_SPECIAL: { JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; - method_and_self[0] = ref_new_not_null(ctx); - method_and_self[1] = ref_new_unknown(ctx); + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); break; } case _WITH_EXCEPT_START: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1736,8 +1736,8 @@ case _PUSH_EXC_INFO: { JitOptRef prev_exc; JitOptRef new_exc; - prev_exc = ref_new_not_null(ctx); - new_exc = ref_new_not_null(ctx); + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -1760,7 +1760,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1780,7 +1780,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1799,7 +1799,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1814,7 +1814,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1834,7 +1834,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1855,8 +1855,8 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1886,12 +1886,12 @@ JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(ref_get_const(ctx, callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } - ref_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); break; } @@ -1913,7 +1913,7 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1925,8 +1925,8 @@ JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - ref_set_null(null); - ref_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } @@ -1935,8 +1935,8 @@ JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1954,12 +1954,12 @@ JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(ref_matches_type(callable, &PyFunction_Type)); - if (ref_is_const(ctx, callable)) { - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(ctx, callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -1993,11 +1993,11 @@ } assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); - if (ref_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { args--; argcount++; } - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -2043,40 +2043,40 @@ case _GUARD_NOS_NULL: { JitOptRef null; null = stack_pointer[-2]; - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); break; } case _GUARD_NOS_NOT_NULL: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_is_not_null(nos)) { + if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_non_null(nos); + sym_set_non_null(nos); break; } case _GUARD_THIRD_NULL: { JitOptRef null; null = stack_pointer[-3]; - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); break; } case _GUARD_CALLABLE_TYPE_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyType_Type); + sym_set_const(callable, (PyObject *)&PyType_Type); break; } @@ -2084,11 +2084,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_has_type(arg)) { - res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); + if (sym_has_type(arg)) { + res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2099,10 +2099,10 @@ case _GUARD_CALLABLE_STR_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyUnicode_Type); + sym_set_const(callable, (PyObject *)&PyUnicode_Type); break; } @@ -2110,11 +2110,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_matches_type(arg, &PyUnicode_Type)) { + if (sym_matches_type(arg, &PyUnicode_Type)) { res = arg; } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2125,10 +2125,10 @@ case _GUARD_CALLABLE_TUPLE_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyTuple_Type); + sym_set_const(callable, (PyObject *)&PyTuple_Type); break; } @@ -2136,11 +2136,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_matches_type(arg, &PyTuple_Type)) { + if (sym_matches_type(arg, &PyTuple_Type)) { res = arg; } else { - res = ref_new_type(ctx, &PyTuple_Type); + res = sym_new_type(ctx, &PyTuple_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2158,8 +2158,8 @@ uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -2183,7 +2183,7 @@ case _CALL_BUILTIN_CLASS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2192,7 +2192,7 @@ case _CALL_BUILTIN_O: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2201,7 +2201,7 @@ case _CALL_BUILTIN_FAST: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2210,7 +2210,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2221,16 +2221,16 @@ JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (ref_get_const(ctx, callable) == len) { + if (sym_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, len); + sym_set_const(callable, len); break; } case _CALL_LEN: { JitOptRef res; - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2241,10 +2241,10 @@ JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (ref_get_const(ctx, callable) == isinstance) { + if (sym_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, isinstance); + sym_set_const(callable, isinstance); break; } @@ -2254,15 +2254,15 @@ JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; - res = ref_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = ref_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); - if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { PyObject *out = Py_False; if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - ref_set_const(res, out); + sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } stack_pointer[-4] = res; @@ -2275,10 +2275,10 @@ JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (ref_get_const(ctx, callable) == list_append) { + if (sym_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, list_append); + sym_set_const(callable, list_append); break; } @@ -2290,7 +2290,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2299,7 +2299,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2308,7 +2308,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2317,7 +2317,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2360,7 +2360,7 @@ case _CALL_KW_NON_PY: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2375,14 +2375,14 @@ case _MAKE_FUNCTION: { JitOptRef func; - func = ref_new_not_null(ctx); + func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { JitOptRef func_out; - func_out = ref_new_not_null(ctx); + func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2394,7 +2394,7 @@ ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); assert(corresponding_check_stack == NULL); assert(co != NULL); int framesize = co->co_framesize; @@ -2413,7 +2413,7 @@ case _BUILD_SLICE: { JitOptRef slice; - slice = ref_new_type(ctx, &PySlice_Type); + slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2422,21 +2422,21 @@ case _CONVERT_VALUE: { JitOptRef result; - result = ref_new_not_null(ctx); + result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2461,38 +2461,38 @@ JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - bool lhs_int = ref_matches_type(lhs, &PyLong_Type); - bool rhs_int = ref_matches_type(rhs, &PyLong_Type); - bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); - bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { if (rhs_float) { - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (lhs_float) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } - else if (!ref_is_const(ctx, rhs)) { - res = ref_new_unknown(ctx); + else if (!sym_is_const(ctx, rhs)) { + res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { - res = ref_new_type(ctx, &PyFloat_Type); + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { + res = sym_new_type(ctx, &PyFloat_Type); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2535,12 +2535,12 @@ case _GUARD_IS_TRUE_POP: { JitOptRef flag; flag = stack_pointer[-1]; - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - ref_set_const(flag, Py_True); + sym_set_const(flag, Py_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2549,12 +2549,12 @@ case _GUARD_IS_FALSE_POP: { JitOptRef flag; flag = stack_pointer[-1]; - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - ref_set_const(flag, Py_False); + sym_set_const(flag, Py_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2563,16 +2563,16 @@ case _GUARD_IS_NONE_POP: { JitOptRef val; val = stack_pointer[-1]; - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - ref_set_const(val, Py_None); + sym_set_const(val, Py_None); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2581,13 +2581,13 @@ case _GUARD_IS_NOT_NONE_POP: { JitOptRef val; val = stack_pointer[-1]; - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } stack_pointer += -1; @@ -2629,7 +2629,7 @@ case _LOAD_CONST_INLINE: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2639,7 +2639,7 @@ case _POP_TOP_LOAD_CONST_INLINE: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } @@ -2647,7 +2647,7 @@ case _LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2675,14 +2675,14 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2692,7 +2692,7 @@ case _POP_CALL_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2702,7 +2702,7 @@ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2712,7 +2712,7 @@ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2722,8 +2722,8 @@ case _LOAD_CONST_UNDER_INLINE: { JitOptRef value; JitOptRef new; - value = ref_new_not_null(ctx); - new = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -2734,8 +2734,8 @@ case _LOAD_CONST_UNDER_INLINE_BORROW: { JitOptRef value; JitOptRef new; - value = ref_new_not_null(ctx); - new = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 994f591c561d6c..e19c0938e20446 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -123,20 +123,20 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_ref_is_bottom(JitOptRef ref) +_Py_uop_sym_is_bottom(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_ref_is_not_null(JitOptRef ref) { +_Py_uop_sym_is_not_null(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -144,7 +144,7 @@ _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return false; } @@ -155,14 +155,14 @@ _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) } bool -_Py_uop_ref_is_null(JitOptRef ref) +_Py_uop_sym_is_null(JitOptRef ref) { return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -170,7 +170,7 @@ _Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return NULL; } @@ -182,7 +182,7 @@ _Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) } void -_Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -233,12 +233,12 @@ _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) } bool -_Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_ref_set_type(ctx, ref, type); + _Py_uop_sym_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -291,7 +291,7 @@ _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int ver } void -_Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -314,12 +314,12 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_ref_tuple_length(ref); + Py_ssize_t len = _Py_uop_sym_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptRef ref_item = _Py_uop_ref_tuple_getitem(ctx, ref, i); + JitOptRef sym_item = _Py_uop_sym_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); - _Py_uop_ref_set_const(ctx, ref_item, item); + _Py_uop_sym_set_const(ctx, sym_item, item); } make_const(sym, const_val); return; @@ -342,29 +342,29 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_ref_is_const(ctx, ref) && - _Py_uop_ref_get_const(ctx, ref) != const_val)) + (_Py_uop_sym_is_const(ctx, ref) && + _Py_uop_sym_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); - PyTypeObject *type = _Py_uop_ref_get_type(value); + PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: if (type == &PyBool_Type) { - _Py_uop_ref_set_const(ctx, value, Py_True); + _Py_uop_sym_set_const(ctx, value, Py_True); } } // value is falsey: else if (type == &PyBool_Type) { - _Py_uop_ref_set_const(ctx, value, Py_False); + _Py_uop_sym_set_const(ctx, value, Py_False); } else if (type == &PyLong_Type) { - _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); + _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); } else if (type == &PyUnicode_Type) { - _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); + _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); } // TODO: More types (GH-130415)! make_const(sym, const_val); @@ -373,7 +373,7 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) } void -_Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { @@ -385,7 +385,7 @@ _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) } void -_Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { @@ -397,7 +397,7 @@ _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) } JitOptRef -_Py_uop_ref_new_unknown(JitOptContext *ctx) +_Py_uop_sym_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -407,7 +407,7 @@ _Py_uop_ref_new_unknown(JitOptContext *ctx) } JitOptRef -_Py_uop_ref_new_not_null(JitOptContext *ctx) +_Py_uop_sym_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -418,20 +418,20 @@ _Py_uop_ref_new_not_null(JitOptContext *ctx) } JitOptRef -_Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) +_Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(res); - _Py_uop_ref_set_type(ctx, ref, typ); + _Py_uop_sym_set_type(ctx, ref, typ); return ref; } // Adds a new reference to const_val, owned by the symbol. JitOptRef -_Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) +_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); @@ -439,24 +439,24 @@ _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(res); - _Py_uop_ref_set_const(ctx, ref, const_val); + _Py_uop_sym_set_const(ctx, ref, const_val); return ref; } JitOptRef -_Py_uop_ref_new_null(JitOptContext *ctx) +_Py_uop_sym_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); - _Py_uop_ref_set_null(ctx, ref); + _Py_uop_sym_set_null(ctx, ref); return ref; } PyTypeObject * -_Py_uop_ref_get_type(JitOptRef ref) +_Py_uop_sym_get_type(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -481,7 +481,7 @@ _Py_uop_ref_get_type(JitOptRef ref) } unsigned int -_Py_uop_ref_get_type_version(JitOptRef ref) +_Py_uop_sym_get_type_version(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -506,26 +506,26 @@ _Py_uop_ref_get_type_version(JitOptRef ref) } bool -_Py_uop_ref_has_type(JitOptRef sym) +_Py_uop_sym_has_type(JitOptRef sym) { - return _Py_uop_ref_get_type(sym) != NULL; + return _Py_uop_sym_get_type(sym) != NULL; } bool -_Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - return _Py_uop_ref_get_type(sym) == typ; + return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version) +_Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) { - return _Py_uop_ref_get_type_version(sym) == version; + return _Py_uop_sym_get_type_version(sym) == version; } int -_Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); switch(sym->tag) { @@ -547,7 +547,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolBorrow(value)); if (truthiness < 0) { return truthiness; @@ -575,7 +575,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) } JitOptRef -_Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -596,24 +596,24 @@ _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) } JitOptRef -_Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple) && item < PyTuple_GET_SIZE(tuple)) { - return _Py_uop_ref_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); + return _Py_uop_sym_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); } - return _Py_uop_ref_new_not_null(ctx); + return _Py_uop_sym_new_not_null(ctx); } int -_Py_uop_ref_tuple_length(JitOptRef ref) +_Py_uop_sym_tuple_length(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -630,7 +630,7 @@ _Py_uop_ref_tuple_length(JitOptRef ref) // Return true if known to be immortal. bool -_Py_uop_sym_is_immortal(JitOptSymbol *sym) +_Py_uop_symbol_is_immortal(JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return _Py_IsImmortal(sym->value.value); @@ -645,14 +645,14 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) } bool -_Py_uop_ref_is_immortal(JitOptRef ref) +_Py_uop_sym_is_immortal(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); - return _Py_uop_sym_is_immortal(sym); + return _Py_uop_symbol_is_immortal(sym); } JitOptRef -_Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); // It's clearer to invert this in the signature: @@ -664,7 +664,7 @@ _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) if (res == NULL) { return out_of_space_ref(ctx); } - int truthiness = _Py_uop_ref_truthiness(ctx, ref); + int truthiness = _Py_uop_sym_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -707,14 +707,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptRef local = _Py_uop_ref_new_unknown(ctx); + JitOptRef local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptRef stackvar = _Py_uop_ref_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -798,44 +798,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptRef ref = _Py_uop_ref_new_unknown(ctx); + JitOptRef ref = _Py_uop_sym_new_unknown(ctx); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "top is NULL"); - TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "top is a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_ref_is_bottom(ref), "top is bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "bottom isn't bottom"); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "int is NULL"); - TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "int is a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(int and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -845,86 +845,86 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_ref_set_const(ctx, ref, val_42); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "42 is NULL"); - TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "42 is not immortal"); + _Py_uop_sym_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "42 is not immortal"); - _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); - _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and float) isn't bottom"); - ref = _Py_uop_ref_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "a bool is not immortal"); + ref = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "a bool is not immortal"); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_ref_set_const(ctx, ref, val_42); - _Py_uop_ref_set_const(ctx, ref, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and 43) isn't bottom"); + _Py_uop_sym_set_const(ctx, ref, val_42); + _Py_uop_sym_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and 43) isn't bottom"); - ref = _Py_uop_ref_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(None) is not False"); - ref = _Py_uop_ref_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(False) is not False"); - ref = _Py_uop_ref_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(0) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(0) is not False"); - JitOptRef i1 = _Py_uop_ref_new_type(ctx, &PyFloat_Type); - JitOptRef i2 = _Py_uop_ref_new_const(ctx, val_43); + JitOptRef i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_sym_new_const(ctx, val_43); JitOptRef array[2] = { i1, i2 }; - ref = _Py_uop_ref_new_tuple(ctx, 2, array); + ref = _Py_uop_sym_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_ref_matches_type(_Py_uop_ref_tuple_getitem(ctx, ref, 0), &PyFloat_Type), + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - ref = _Py_uop_ref_new_const(ctx, tuple); + ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - ref = _Py_uop_ref_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_sym_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_ref_is_not_null(_Py_uop_ref_tuple_getitem(ctx, ref, 42)), + _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptRef value = _Py_uop_ref_new_type(ctx, &PyBool_Type); - ref = _Py_uop_ref_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "truthiness is not NULL"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == false, "value is constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_ref_set_const(ctx, ref, Py_False); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == Py_False, "truthiness is not False"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == true, "value is not constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == Py_True, "value is not True"); + JitOptRef value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + ref = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "truthiness is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); + _Py_uop_sym_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 39ffc1d7e0cbd6..ed03c316a54ae6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -735,7 +735,7 @@ def visit(stmt: Stmt) -> None: continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_", "ref_", "PyJitRef")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index dd8420b792f53e..95ccc39af1f4a0 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -124,15 +124,15 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: local.in_local = True if var.is_array(): if var.size == "1": - out.emit(f"{var.name}[0] = ref_new_not_null(ctx);\n") + out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") else: out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = ref_new_not_null(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = ref_new_null(ctx);\n") + out.emit(f"{var.name} = sym_new_null(ctx);\n") else: - out.emit(f"{var.name} = ref_new_not_null(ctx);\n") + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") class OptimizerEmitter(Emitter): From 2d779c4ded74b3287f6ea72ef8795e9dde126146 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 23:11:37 +0800 Subject: [PATCH 21/24] rename jitref functions --- Include/internal/pycore_optimizer.h | 11 ++--- Python/optimizer_symbols.c | 65 +++++++++++++++-------------- 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 88acea076dbc80..07c33f9183c6ca 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -230,24 +230,19 @@ typedef union { #define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) static inline JitOptSymbol * -PyJitRef_AsSymbolBorrow(JitOptRef ref) +PyJitRef_Unwrap(JitOptRef ref) { return JIT_BITS_TO_PTR_MASKED(ref); } bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); -static inline JitOptRef -PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) -{ - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; -} static inline JitOptRef -PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +PyJitRef_Wrap(JitOptSymbol *sym) { if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return PyJitRef_FromSymbolBorrow(sym); + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; } return (JitOptRef){.bits=(uintptr_t)sym}; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e19c0938e20446..838f25ad288ab0 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -91,7 +91,7 @@ out_of_space(JitOptContext *ctx) JitOptRef out_of_space_ref(JitOptContext *ctx) { - return PyJitRef_FromSymbolSteal(out_of_space(ctx)); + return PyJitRef_Wrap(out_of_space(ctx)); } static JitOptSymbol * @@ -125,26 +125,26 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) bool _Py_uop_sym_is_bottom(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool _Py_uop_sym_is_not_null(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return false; } @@ -157,20 +157,20 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) bool _Py_uop_sym_is_null(JitOptRef ref) { - return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; + return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * _Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return NULL; } @@ -184,7 +184,7 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -235,7 +235,7 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { _Py_uop_sym_set_type(ctx, ref, type); @@ -293,7 +293,7 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int ver void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -348,7 +348,8 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) sym_set_bottom(ctx, sym); return; } - JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); + JitOptRef value = PyJitRef_Wrap( + allocation_base(ctx) + sym->truthiness.value); PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: @@ -375,7 +376,7 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -387,7 +388,7 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -403,7 +404,7 @@ _Py_uop_sym_new_unknown(JitOptContext *ctx) if (res == NULL) { return out_of_space_ref(ctx); } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef @@ -414,7 +415,7 @@ _Py_uop_sym_new_not_null(JitOptContext *ctx) return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef @@ -424,7 +425,7 @@ _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) if (res == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(res); + JitOptRef ref = PyJitRef_Wrap(res); _Py_uop_sym_set_type(ctx, ref, typ); return ref; } @@ -438,7 +439,7 @@ _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) if (res == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(res); + JitOptRef ref = PyJitRef_Wrap(res); _Py_uop_sym_set_const(ctx, ref, const_val); return ref; } @@ -450,7 +451,7 @@ _Py_uop_sym_new_null(JitOptContext *ctx) if (null_sym == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); + JitOptRef ref = PyJitRef_Wrap(null_sym); _Py_uop_sym_set_null(ctx, ref); return ref; } @@ -458,7 +459,7 @@ _Py_uop_sym_new_null(JitOptContext *ctx) PyTypeObject * _Py_uop_sym_get_type(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -483,7 +484,7 @@ _Py_uop_sym_get_type(JitOptRef ref) unsigned int _Py_uop_sym_get_type_version(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -527,7 +528,7 @@ _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -548,7 +549,7 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; int truthiness = _Py_uop_sym_truthiness(ctx, - PyJitRef_FromSymbolBorrow(value)); + PyJitRef_Wrap(value)); if (truthiness < 0) { return truthiness; } @@ -589,16 +590,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(PyJitRef_AsSymbolBorrow(args[i]) - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx)); } } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; @@ -607,7 +608,7 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); + return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]); } return _Py_uop_sym_new_not_null(ctx); } @@ -615,7 +616,7 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) int _Py_uop_sym_tuple_length(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -647,14 +648,14 @@ _Py_uop_symbol_is_immortal(JitOptSymbol *sym) bool _Py_uop_sym_is_immortal(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return _Py_uop_symbol_is_immortal(sym); } JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { - JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *value = PyJitRef_Unwrap(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { @@ -673,7 +674,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } // 0 on success, -1 on error. @@ -809,7 +810,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); - ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); + ref = PyJitRef_Wrap(make_bottom(ctx)); if (PyJitRef_IsNull(ref)) { goto fail; } From b74e160f9d0810732b470ce707615d2a2810ccbc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 5 Jun 2025 01:33:37 +0800 Subject: [PATCH 22/24] Address review --- Include/internal/pycore_optimizer.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 07c33f9183c6ca..6fd6b876224b70 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -226,8 +226,9 @@ typedef union { uintptr_t bits; } JitOptRef; -#define JIT_BITS_TO_PTR(REF) ((JitOptSymbol *)((REF).bits)) -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) +#define REF_IS_BORROWED 1 + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -242,7 +243,7 @@ static inline JitOptRef PyJitRef_Wrap(JitOptSymbol *sym) { if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + return (JitOptRef){.bits=(uintptr_t)sym | REF_IS_BORROWED}; } return (JitOptRef){.bits=(uintptr_t)sym}; } @@ -250,14 +251,10 @@ PyJitRef_Wrap(JitOptSymbol *sym) static inline JitOptRef PyJitRef_Borrow(JitOptRef ref) { - return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; + return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; } -#ifndef Py_GIL_DISABLED -static const JitOptRef PyJitRef_NULL = {.bits = PyStackRef_NULL_BITS}; -#else -static const JitOptRef PyJitRef_NULL = {.bits = Py_TAG_DEFERRED}; -#endif +static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; static inline bool PyJitRef_IsNull(JitOptRef ref) @@ -268,7 +265,7 @@ PyJitRef_IsNull(JitOptRef ref) static inline int PyJitRef_IsBorrowed(JitOptRef ref) { - return (ref.bits & Py_TAG_REFCNT) == Py_TAG_REFCNT; + return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; } struct _Py_UOpsAbstractFrame { From 3ebcc2013124f2a3163934ec6928e97b10fcffbc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 5 Jun 2025 01:35:02 +0800 Subject: [PATCH 23/24] Update comment --- Python/optimizer_bytecodes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 7d65a9187b2497..4f8197db208d6e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -296,7 +296,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } @@ -320,7 +320,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } @@ -344,7 +344,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } From 914f1ff84c51042a4e010155aefb1c049f16aa5d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:43:19 +0800 Subject: [PATCH 24/24] Fix changes from upstream (no more casts) --- Include/internal/pycore_optimizer.h | 3 --- Python/optimizer_bytecodes.c | 20 ++++++++++---------- Python/optimizer_cases.c.h | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 6fd6b876224b70..b27341cb1dbbeb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -242,9 +242,6 @@ bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); static inline JitOptRef PyJitRef_Wrap(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | REF_IS_BORROWED}; - } return (JitOptRef){.bits=(uintptr_t)sym}; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3d8d652bb5075e..07b05fc3323ece 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -341,7 +341,7 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -666,7 +666,7 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -733,9 +733,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } @@ -754,11 +754,11 @@ dummy_func(void) { break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -770,7 +770,7 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; } @@ -837,13 +837,13 @@ dummy_func(void) { } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } @@ -863,7 +863,7 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b86d73f39a362a..354331ef618f67 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -695,7 +695,7 @@ case _BINARY_OP_SUBSCR_INIT_CALL: { JitOptRef new_frame; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -809,7 +809,7 @@ case _SEND_GEN_FRAME: { JitOptRef gen_frame; - gen_frame = NULL; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = gen_frame; break; @@ -1305,7 +1305,7 @@ JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1665,7 +1665,7 @@ case _FOR_ITER_GEN_FRAME: { JitOptRef gen_frame; - gen_frame = NULL; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -1844,7 +1844,7 @@ ctx->done = true; break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1977,9 +1977,9 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -1993,7 +1993,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); @@ -2149,7 +2149,7 @@ case _CREATE_INIT_FRAME: { JitOptRef init_frame; - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2316,7 +2316,7 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg;