From 6097656b7d86be5e813079c09a2b7c84015d670e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 13:57:44 +0000 Subject: [PATCH 1/8] Add stats for watched dict modifications --- Include/cpython/pystats.h | 2 ++ Include/internal/pycore_dict.h | 1 + Python/specialize.c | 1 + Tools/scripts/summarize_stats.py | 2 +- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index bf0cfe4cb695b4..54a8f861613041 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -133,6 +133,8 @@ typedef struct _rare_event_stats { uint64_t builtin_dict; /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ uint64_t func_modification; + /* Modifying a dict that is being watched */ + uint64_t watched_dict_modification; } RareEventStats; typedef struct _stats { diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 233da058f464d1..7a519411dcbe83 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -236,6 +236,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, assert(Py_REFCNT((PyObject*)mp) > 0); int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { + RARE_EVENT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); return DICT_NEXT_VERSION(interp) | watcher_bits; } diff --git a/Python/specialize.c b/Python/specialize.c index e38e3556a6d642..1346d57233588b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -275,6 +275,7 @@ print_rare_event_stats(FILE *out, RareEventStats *stats) fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); } static void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9b7e7b999ea7c7..8caa5faf0f70e5 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -416,7 +416,7 @@ def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ (key[len(prefix) + 1:-1], val) - for key, val in self._data.items() + for key.replace("_", " "), val in self._data.items() if key.startswith(prefix) ] From 7b9e10f33b55b978eb7dae5030f40b4183df3e9e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 14:11:09 +0000 Subject: [PATCH 2/8] Increase stats not event counter --- Include/internal/pycore_dict.h | 2 +- Tools/scripts/summarize_stats.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 7a519411dcbe83..691a394f3c58ec 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -236,7 +236,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, assert(Py_REFCNT((PyObject*)mp) > 0); int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; if (watcher_bits) { - RARE_EVENT_INC(watched_dict_modification); + RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); return DICT_NEXT_VERSION(interp) | watcher_bits; } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 8caa5faf0f70e5..7891b9cf923d33 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -415,8 +415,8 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ - (key[len(prefix) + 1:-1], val) - for key.replace("_", " "), val in self._data.items() + (key[len(prefix) + 1:-1].replace("_", " "), val) + for key, val in self._data.items() if key.startswith(prefix) ] From fe6f94a21375e2403eccc237f9458083d55c0c43 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 16:56:40 +0000 Subject: [PATCH 3/8] Always unwatch dict when global watcher is called --- Python/optimizer_analysis.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 2cfbf4b349d0f5..94f18aa08b53ec 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -28,6 +28,11 @@ increment_mutations(PyObject* dict) { d->ma_version_tag += (1 << DICT_MAX_WATCHERS); } +/* The first two dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 + static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value) @@ -40,13 +45,10 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); increment_mutations(dict); } - else { - PyDict_Unwatch(1, dict); - } + PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); return 0; } - static void global_to_const(_PyUOpInstruction *inst, PyObject *obj) { @@ -82,11 +84,6 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 0; } -/* The first two dict watcher IDs are reserved for CPython, - * so we don't need to check that they haven't been used */ -#define BUILTINS_WATCHER_ID 0 -#define GLOBALS_WATCHER_ID 1 - /* Returns 1 if successfully optimized * 0 if the trace is not suitable for optimization (yet) * -1 if there was an error. */ From 7726b101d081fc61f94a07d4c77f4f1769172aba Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 20:54:23 +0000 Subject: [PATCH 4/8] Stop watching builtins after max modifications --- Python/pylifecycle.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0cac7109340129..8c50019dbe37d4 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -611,9 +611,12 @@ static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (event != PyDict_EVENT_CLONED && interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { _Py_Executors_InvalidateAll(interp); } + else { + PyDict_Unwatch(0, interp->builtins); + } RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } From 89841a342af380d6ccb491a0de86ce25bb3f1f71 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 20:55:07 +0000 Subject: [PATCH 5/8] Respond to 'clone' events --- Python/optimizer_analysis.c | 3 --- Python/pylifecycle.c | 3 --- 2 files changed, 6 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 94f18aa08b53ec..0f823867587c26 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -37,9 +37,6 @@ static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value) { - if (event == PyDict_EVENT_CLONED) { - return 0; - } uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8c50019dbe37d4..288edd06356df4 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -614,9 +614,6 @@ builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, Py if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { _Py_Executors_InvalidateAll(interp); } - else { - PyDict_Unwatch(0, interp->builtins); - } RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } From 482aac388208d0a9e5965c5eb87c2f5f26bf420f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 21:05:07 +0000 Subject: [PATCH 6/8] Add counter for watched globals --- Include/cpython/pystats.h | 1 + Python/optimizer_analysis.c | 1 + Python/specialize.c | 1 + 3 files changed, 3 insertions(+) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 54a8f861613041..0f50439b73848e 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -135,6 +135,7 @@ typedef struct _rare_event_stats { uint64_t func_modification; /* Modifying a dict that is being watched */ uint64_t watched_dict_modification; + uint64_t watched_globals_modification; } RareEventStats; typedef struct _stats { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0f823867587c26..c7ba55c7a3e2ec 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -38,6 +38,7 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value) { uint64_t watched_mutations = get_mutations(dict); + RARE_EVENT_STAT_INC(watched_globals_modification); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); increment_mutations(dict); diff --git a/Python/specialize.c b/Python/specialize.c index 1346d57233588b..ea2638570f22d0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -276,6 +276,7 @@ print_rare_event_stats(FILE *out, RareEventStats *stats) fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); } static void From 247cd3bbb3db179b9706ac8d9dc07732d16a4a68 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 04:12:51 +0000 Subject: [PATCH 7/8] Make sure watched modification count is preserved when update dict version tag. --- Include/internal/pycore_dict.h | 4 ++-- Python/optimizer_analysis.c | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 691a394f3c58ec..0ebe701bc16f81 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -209,6 +209,7 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) +#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) #ifdef Py_GIL_DISABLED #define DICT_NEXT_VERSION(INTERP) \ @@ -238,9 +239,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(interp); + return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index c7ba55c7a3e2ec..b393b805184749 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -37,13 +37,16 @@ static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value) { - uint64_t watched_mutations = get_mutations(dict); + assert(((PyDictObject *)dict)->ma_version_tag & 2); + int watched_mutations = get_mutations(dict); + printf("globals_watcher_callback called. dict %p, %d\n", dict, watched_mutations); RARE_EVENT_STAT_INC(watched_globals_modification); - if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); - increment_mutations(dict); - } + assert(watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); + assert((((PyDictObject *)dict)->ma_version_tag & DICT_WATCHER_MASK) == 0); + increment_mutations(dict); + assert(get_mutations(dict) == watched_mutations + 1); return 0; } @@ -112,8 +115,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, uint32_t builtins_watched = 0; uint32_t globals_checked = 0; uint32_t globals_watched = 0; - if (interp->dict_state.watchers[1] == NULL) { - interp->dict_state.watchers[1] = globals_watcher_callback; + if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { + interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; } for (int pc = 0; pc < buffer_size; pc++) { _PyUOpInstruction *inst = &buffer[pc]; From 10542025614f6b15289c8d96ce5d99bb39537191 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 04:18:31 +0000 Subject: [PATCH 8/8] Remove debug code --- Python/optimizer_analysis.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b393b805184749..b14e6950b4a06b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -37,16 +37,11 @@ static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value) { - assert(((PyDictObject *)dict)->ma_version_tag & 2); - int watched_mutations = get_mutations(dict); - printf("globals_watcher_callback called. dict %p, %d\n", dict, watched_mutations); RARE_EVENT_STAT_INC(watched_globals_modification); - assert(watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); + assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); - PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); - assert((((PyDictObject *)dict)->ma_version_tag & DICT_WATCHER_MASK) == 0); increment_mutations(dict); - assert(get_mutations(dict) == watched_mutations + 1); + PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); return 0; }