Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b973188

Browse files
DinoVDinoV
committed
Lock on allocation, use version as sync point
1 parent be10b2d commit b973188

11 files changed

+179
-117
lines changed

‎Include/cpython/pyatomic.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,10 +465,16 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
465465
static inline void *
466466
_Py_atomic_load_ptr_acquire(const void *obj);
467467

468+
static inline uintptr_t
469+
_Py_atomic_load_uintptr_acquire(const uintptr_t *obj);
470+
468471
// Stores `*obj = value` (release operation)
469472
static inline void
470473
_Py_atomic_store_ptr_release(void *obj, void *value);
471474

475+
static inline void
476+
_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
477+
472478
static inline void
473479
_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
474480

@@ -491,6 +497,8 @@ static inline Py_ssize_t
491497
_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);
492498

493499

500+
501+
494502
// --- _Py_atomic_fence ------------------------------------------------------
495503

496504
// Sequential consistency fence. C11 fences have complex semantics. When

‎Include/cpython/pyatomic_gcc.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -492,10 +492,18 @@ static inline void *
492492
_Py_atomic_load_ptr_acquire(const void *obj)
493493
{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); }
494494

495+
static inline uintptr_t
496+
_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
497+
{ return (uintptr_t)__atomic_load_n((uintptr_t *)obj, __ATOMIC_ACQUIRE); }
498+
495499
static inline void
496500
_Py_atomic_store_ptr_release(void *obj, void *value)
497501
{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); }
498502

503+
static inline void
504+
_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
505+
{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
506+
499507
static inline void
500508
_Py_atomic_store_int_release(int *obj, int value)
501509
{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }

‎Include/cpython/pyatomic_msc.h

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -914,6 +914,18 @@ _Py_atomic_load_ptr_acquire(const void *obj)
914914
#endif
915915
}
916916

917+
static inline uintptr_t
918+
_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
919+
{
920+
#if defined(_M_X64) || defined(_M_IX86)
921+
return *(uintptr_t volatile *)obj;
922+
#elif defined(_M_ARM64)
923+
return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);
924+
#else
925+
# error "no implementation of _Py_atomic_load_ptr_acquire"
926+
#endif
927+
}
928+
917929
static inline void
918930
_Py_atomic_store_ptr_release(void *obj, void *value)
919931
{
@@ -926,6 +938,19 @@ _Py_atomic_store_ptr_release(void *obj, void *value)
926938
#endif
927939
}
928940

941+
static inline void
942+
_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
943+
{
944+
#if defined(_M_X64) || defined(_M_IX86)
945+
*(uintptr_t volatile *)obj = value;
946+
#elif defined(_M_ARM64)
947+
_Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
948+
__stlr32((unsigned __int64 volatile *)obj, (unsigned __int64)value);
949+
#else
950+
# error "no implementation of _Py_atomic_store_int_release"
951+
#endif
952+
}
953+
929954
static inline void
930955
_Py_atomic_store_int_release(int *obj, int value)
931956
{

‎Include/cpython/pyatomic_std.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -863,6 +863,14 @@ _Py_atomic_load_ptr_acquire(const void *obj)
863863
memory_order_acquire);
864864
}
865865

866+
static inline uintptr_t
867+
_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
868+
{
869+
_Py_USING_STD;
870+
return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
871+
memory_order_acquire);
872+
}
873+
866874
static inline void
867875
_Py_atomic_store_ptr_release(void *obj, void *value)
868876
{
@@ -871,6 +879,14 @@ _Py_atomic_store_ptr_release(void *obj, void *value)
871879
memory_order_release);
872880
}
873881

882+
static inline void
883+
_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
884+
{
885+
_Py_USING_STD;
886+
atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
887+
memory_order_release);
888+
}
889+
874890
static inline void
875891
_Py_atomic_store_int_release(int *obj, int value)
876892
{

‎Include/internal/pycore_gc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
9292
* threads and needs special purpose when freeing due to
9393
* the possibility of in-flight lock-free reads occurring.
9494
* Objects with this bit that are GC objects will automatically
95-
* delay-freed by PyObject_GC_Del. */
95+
* delay-freed by PyObject_GC_Del. */
9696
static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
9797
return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
9898
}

‎Include/internal/pycore_pyatomic_ft_wrappers.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,14 @@ extern "C" {
2525
_Py_atomic_load_ssize_relaxed(&value)
2626
#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \
2727
_Py_atomic_load_ptr_acquire(&value)
28+
#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \
29+
_Py_atomic_load_uintptr_acquire(&value)
2830
#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \
2931
_Py_atomic_store_ptr_relaxed(&value, new_value)
3032
#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \
3133
_Py_atomic_store_ptr_release(&value, new_value)
34+
#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \
35+
_Py_atomic_store_uintptr_release(&value, new_value)
3236
#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \
3337
_Py_atomic_store_ssize_relaxed(&value, new_value)
3438
#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \
@@ -37,8 +41,10 @@ extern "C" {
3741
#define FT_ATOMIC_LOAD_SSIZE(value) value
3842
#define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value
3943
#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value
44+
#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value
4045
#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value
4146
#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value
47+
#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value
4248
#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value
4349
#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value
4450

‎Python/bytecodes.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "pycore_object.h" // _PyObject_GC_TRACK()
1919
#include "pycore_opcode_metadata.h" // uop names
2020
#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
21+
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE
2122
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
2223
#include "pycore_pystate.h" // _PyInterpreterState_GET()
2324
#include "pycore_range.h" // _PyRangeIterObject
@@ -148,10 +149,11 @@ dummy_func(
148149
uintptr_t global_version =
149150
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
150151
~_PY_EVAL_EVENTS_MASK;
151-
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
152+
PyCodeObject *code = _PyFrame_GetCode(frame);
153+
uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
152154
assert((code_version & 255) == 0);
153155
if (code_version != global_version) {
154-
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
156+
int err = _Py_Instrument(code, tstate->interp);
155157
ERROR_IF(err, error);
156158
next_instr = this_instr;
157159
}

‎Python/ceval.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "pycore_opcode_metadata.h" // EXTRA_CASES
1919
#include "pycore_optimizer.h" // _PyUOpExecutor_Type
2020
#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
21+
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE
2122
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
2223
#include "pycore_pystate.h" // _PyInterpreterState_GET()
2324
#include "pycore_range.h" // _PyRangeIterObject

‎Python/generated_cases.c.h

Lines changed: 3 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)