Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 336366f

Browse files
brandtbucherpablogsal
brandtbucher
and
pablogsal
authored
GH-140643: Add <native> and <GC> frames to the sampling profiler (#141108)
- Introduce a new field in the GC state to store the frame that initiated garbage collection. - Update RemoteUnwinder to include options for including "<native>" and "<GC>" frames in the stack trace. - Modify the sampling profiler to accept parameters for controlling the inclusion of native and GC frames. - Enhance the stack collector to properly format and append these frames during profiling. - Add tests to verify the correct behavior of the profiler with respect to native and GC frames, including options to exclude them. Co-authored-by: Pablo Galindo Salgado <[email protected]>
1 parent 89a914c commit 336366f

18 files changed

Lines changed: 463 additions & 84 deletions

‎Doc/library/profile.rst‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,14 @@ Profile with real-time sampling statistics::
265265

266266
Sample all threads in the process instead of just the main thread
267267

268+
.. option:: --native
269+
270+
Include artificial ``<native>`` frames to denote calls to non-Python code.
271+
272+
.. option:: --no-gc
273+
274+
Don't include artificial ``<GC>`` frames to denote active garbage collection.
275+
268276
.. option:: --realtime-stats
269277

270278
Print real-time sampling statistics during profiling
@@ -349,7 +357,7 @@ This section documents the programmatic interface for the :mod:`!profiling.sampl
349357
For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information
350358
about statistical profiling, see :ref:`statistical-profiling`
351359

352-
.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False)
360+
.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False, native=False, gc=True)
353361

354362
Sample a Python process and generate profiling data.
355363

@@ -367,6 +375,8 @@ about statistical profiling, see :ref:`statistical-profiling`
367375
:param bool show_summary: Whether to show summary statistics (default: True)
368376
:param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats')
369377
:param bool realtime_stats: Whether to display real-time statistics (default: False)
378+
:param bool native: Whether to include ``<native>`` frames (default: False)
379+
:param bool gc: Whether to include ``<GC>`` frames (default: True)
370380

371381
:raises ValueError: If output_format is not 'pstats' or 'collapsed'
372382

‎Include/internal/pycore_debug_offsets.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ typedef struct _Py_DebugOffsets {
212212
struct _gc {
213213
uint64_t size;
214214
uint64_t collecting;
215+
uint64_t frame;
215216
} gc;
216217

217218
// Generator object offset;
@@ -355,6 +356,7 @@ typedef struct _Py_DebugOffsets {
355356
.gc = { \
356357
.size = sizeof(struct _gc_runtime_state), \
357358
.collecting = offsetof(struct _gc_runtime_state, collecting), \
359+
.frame = offsetof(struct _gc_runtime_state, frame), \
358360
}, \
359361
.gen_object = { \
360362
.size = sizeof(PyGenObject), \

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,12 @@ struct _Py_global_strings {
4646
STRUCT_FOR_STR(dot_locals, ".<locals>")
4747
STRUCT_FOR_STR(empty, "")
4848
STRUCT_FOR_STR(format, ".format")
49+
STRUCT_FOR_STR(gc, "<GC>")
4950
STRUCT_FOR_STR(generic_base, ".generic_base")
5051
STRUCT_FOR_STR(json_decoder, "json.decoder")
5152
STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
5253
STRUCT_FOR_STR(list_err, "list index out of range")
54+
STRUCT_FOR_STR(native, "<native>")
5355
STRUCT_FOR_STR(str_replace_inf, "1e309")
5456
STRUCT_FOR_STR(type_params, ".type_params")
5557
STRUCT_FOR_STR(utf_8, "utf-8")
@@ -486,6 +488,7 @@ struct _Py_global_strings {
486488
STRUCT_FOR_ID(fullerror)
487489
STRUCT_FOR_ID(func)
488490
STRUCT_FOR_ID(future)
491+
STRUCT_FOR_ID(gc)
489492
STRUCT_FOR_ID(generation)
490493
STRUCT_FOR_ID(get)
491494
STRUCT_FOR_ID(get_debug)
@@ -629,6 +632,7 @@ struct _Py_global_strings {
629632
STRUCT_FOR_ID(name_from)
630633
STRUCT_FOR_ID(namespace_separator)
631634
STRUCT_FOR_ID(namespaces)
635+
STRUCT_FOR_ID(native)
632636
STRUCT_FOR_ID(ndigits)
633637
STRUCT_FOR_ID(nested)
634638
STRUCT_FOR_ID(new_file_name)

‎Include/internal/pycore_interp_structs.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,9 @@ struct _gc_runtime_state {
212212
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
213213
/* true if we are currently running the collector */
214214
int collecting;
215+
// The frame that started the current collection. It might be NULL even when
216+
// collecting (if no Python frame is running):
217+
_PyInterpreterFrame *frame;
215218
/* list of uncollectable objects */
216219
PyObject *garbage;
217220
/* a list of callbacks to be invoked when collection is performed */

‎Include/internal/pycore_interpframe_structs.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ enum _frameowner {
2424
FRAME_OWNED_BY_GENERATOR = 1,
2525
FRAME_OWNED_BY_FRAME_OBJECT = 2,
2626
FRAME_OWNED_BY_INTERPRETER = 3,
27-
FRAME_OWNED_BY_CSTACK = 4,
2827
};
2928

3029
struct _PyInterpreterFrame {

‎Include/internal/pycore_runtime_init_generated.h‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_unicodeobject_generated.h‎

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/profiling/sampling/flamegraph.js‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -151,17 +151,22 @@ function createPythonTooltip(data) {
151151
const funcname = resolveString(d.data.funcname) || resolveString(d.data.name);
152152
const filename = resolveString(d.data.filename) || "";
153153

154+
// Don't show file location for special frames like <GC> and <native>
155+
const isSpecialFrame = filename === "~";
156+
const fileLocationHTML = isSpecialFrame ? "" : `
157+
<div style="color: #5a6c7d; font-size: 13px; margin-bottom: 12px;
158+
font-family: monospace; background: #f8f9fa;
159+
padding: 4px 8px; border-radius: 4px; word-break: break-all; overflow-wrap: break-word;">
160+
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
161+
</div>`;
162+
154163
const tooltipHTML = `
155164
<div>
156165
<div style="color: #3776ab; font-weight: 600; font-size: 16px;
157166
margin-bottom: 8px; line-height: 1.3; word-break: break-word; overflow-wrap: break-word;">
158167
${funcname}
159168
</div>
160-
<div style="color: #5a6c7d; font-size: 13px; margin-bottom: 12px;
161-
font-family: monospace; background: #f8f9fa;
162-
padding: 4px 8px; border-radius: 4px; word-break: break-all; overflow-wrap: break-word;">
163-
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
164-
</div>
169+
${fileLocationHTML}
165170
<div style="display: grid; grid-template-columns: auto 1fr;
166171
gap: 8px 16px; font-size: 14px;">
167172
<span style="color: #5a6c7d; font-weight: 500;">Execution Time:</span>
@@ -474,14 +479,23 @@ function populateStats(data) {
474479
if (i < hotSpots.length && hotSpots[i]) {
475480
const hotspot = hotSpots[i];
476481
const filename = hotspot.filename || 'unknown';
477-
const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown';
478482
const lineno = hotspot.lineno ?? '?';
479483
let funcDisplay = hotspot.funcname || 'unknown';
480484
if (funcDisplay.length > 35) {
481485
funcDisplay = funcDisplay.substring(0, 32) + '...';
482486
}
483487

484-
document.getElementById(`hotspot-file-${num}`).textContent = `${basename}:${lineno}`;
488+
// Don't show file:line for special frames like <GC> and <native>
489+
const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?');
490+
let fileDisplay;
491+
if (isSpecialFrame) {
492+
fileDisplay = '--';
493+
} else {
494+
const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown';
495+
fileDisplay = `${basename}:${lineno}`;
496+
}
497+
498+
document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay;
485499
document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay;
486500
document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`;
487501
} else {

‎Lib/profiling/sampling/sample.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,19 +137,19 @@ def _run_with_sync(original_cmd):
137137

138138

139139
class SampleProfiler:
140-
def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True):
140+
def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True):
141141
self.pid = pid
142142
self.sample_interval_usec = sample_interval_usec
143143
self.all_threads = all_threads
144144
if _FREE_THREADED_BUILD:
145145
self.unwinder = _remote_debugging.RemoteUnwinder(
146-
self.pid, all_threads=self.all_threads, mode=mode,
146+
self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc,
147147
skip_non_matching_threads=skip_non_matching_threads
148148
)
149149
else:
150150
only_active_threads = bool(self.all_threads)
151151
self.unwinder = _remote_debugging.RemoteUnwinder(
152-
self.pid, only_active_thread=only_active_threads, mode=mode,
152+
self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc,
153153
skip_non_matching_threads=skip_non_matching_threads
154154
)
155155
# Track sample intervals and total sample count
@@ -616,6 +616,8 @@ def sample(
616616
output_format="pstats",
617617
realtime_stats=False,
618618
mode=PROFILING_MODE_WALL,
619+
native=False,
620+
gc=True,
619621
):
620622
# PROFILING_MODE_ALL implies no skipping at all
621623
if mode == PROFILING_MODE_ALL:
@@ -627,7 +629,7 @@ def sample(
627629
skip_idle = mode != PROFILING_MODE_WALL
628630

629631
profiler = SampleProfiler(
630-
pid, sample_interval_usec, all_threads=all_threads, mode=mode,
632+
pid, sample_interval_usec, all_threads=all_threads, mode=mode, native=native, gc=gc,
631633
skip_non_matching_threads=skip_non_matching_threads
632634
)
633635
profiler.realtime_stats = realtime_stats
@@ -717,6 +719,8 @@ def wait_for_process_and_sample(pid, sort_value, args):
717719
output_format=args.format,
718720
realtime_stats=args.realtime_stats,
719721
mode=mode,
722+
native=args.native,
723+
gc=args.gc,
720724
)
721725

722726

@@ -767,9 +771,19 @@ def main():
767771
sampling_group.add_argument(
768772
"--realtime-stats",
769773
action="store_true",
770-
default=False,
771774
help="Print real-time sampling statistics (Hz, mean, min, max, stdev) during profiling",
772775
)
776+
sampling_group.add_argument(
777+
"--native",
778+
action="store_true",
779+
help="Include artificial \"<native>\" frames to denote calls to non-Python code.",
780+
)
781+
sampling_group.add_argument(
782+
"--no-gc",
783+
action="store_false",
784+
dest="gc",
785+
help="Don't include artificial \"<GC>\" frames to denote active garbage collection.",
786+
)
773787

774788
# Mode options
775789
mode_group = parser.add_argument_group("Mode options")
@@ -934,6 +948,8 @@ def main():
934948
output_format=args.format,
935949
realtime_stats=args.realtime_stats,
936950
mode=mode,
951+
native=args.native,
952+
gc=args.gc,
937953
)
938954
elif args.module or args.args:
939955
if args.module:

0 commit comments

Comments
 (0)