Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dac78a5

Browse files
pablogsalpablogsal
committed
Merge remote-tracking branch 'upstream/main' into tachyon-opcodes
# Conflicts: # Lib/test/test_profiling/test_sampling_profiler/test_integration.py
2 parents 965f521 + ef51a7c commit dac78a5

27 files changed

Lines changed: 627 additions & 397 deletions

‎Doc/library/exceptions.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -978,6 +978,12 @@ their subgroups based on the types of the contained exceptions.
978978
raises a :exc:`TypeError` if any contained exception is not an
979979
:exc:`Exception` subclass.
980980

981+
.. impl-detail::
982+
983+
The ``excs`` parameter may be any sequence, but lists and tuples are
984+
specifically processed more efficiently here. For optimal performance,
985+
pass a tuple as ``excs``.
986+
981987
.. attribute:: message
982988

983989
The ``msg`` argument to the constructor. This is a read-only attribute.

‎Include/cpython/pyerrors.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ typedef struct {
1818
PyException_HEAD
1919
PyObject *msg;
2020
PyObject *excs;
21+
PyObject *excs_str;
2122
} PyBaseExceptionGroupObject;
2223

2324
typedef struct {

‎Include/cpython/pystate.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,13 @@ struct _ts {
135135
/* Pointer to currently executing frame. */
136136
struct _PyInterpreterFrame *current_frame;
137137

138+
/* Pointer to the base frame (bottommost sentinel frame).
139+
Used by profilers to validate complete stack unwinding.
140+
Points to the embedded base_frame in _PyThreadStateImpl.
141+
The frame is embedded there rather than here because _PyInterpreterFrame
142+
is defined in internal headers that cannot be exposed in the public API. */
143+
struct _PyInterpreterFrame *base_frame;
144+
138145
struct _PyInterpreterFrame *last_profiled_frame;
139146

140147
Py_tracefunc c_profilefunc;

‎Include/internal/pycore_debug_offsets.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ typedef struct _Py_DebugOffsets {
102102
uint64_t next;
103103
uint64_t interp;
104104
uint64_t current_frame;
105+
uint64_t base_frame;
105106
uint64_t last_profiled_frame;
106107
uint64_t thread_id;
107108
uint64_t native_thread_id;
@@ -273,6 +274,7 @@ typedef struct _Py_DebugOffsets {
273274
.next = offsetof(PyThreadState, next), \
274275
.interp = offsetof(PyThreadState, interp), \
275276
.current_frame = offsetof(PyThreadState, current_frame), \
277+
.base_frame = offsetof(PyThreadState, base_frame), \
276278
.last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \
277279
.thread_id = offsetof(PyThreadState, thread_id), \
278280
.native_thread_id = offsetof(PyThreadState, native_thread_id), \

‎Include/internal/pycore_tstate.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ extern "C" {
1010

1111
#include "pycore_brc.h" // struct _brc_thread_state
1212
#include "pycore_freelist_state.h" // struct _Py_freelists
13+
#include "pycore_interpframe_structs.h" // _PyInterpreterFrame
1314
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
1415
#include "pycore_qsbr.h" // struct qsbr
1516
#include "pycore_uop.h" // struct _PyUOpInstruction
@@ -61,6 +62,10 @@ typedef struct _PyThreadStateImpl {
6162
// semi-public fields are in PyThreadState.
6263
PyThreadState base;
6364

65+
// Embedded base frame - sentinel at the bottom of the frame stack.
66+
// Used by profiling/sampling to detect incomplete stack traces.
67+
_PyInterpreterFrame base_frame;
68+
6469
// The reference count field is used to synchronize deallocation of the
6570
// thread state during runtime finalization.
6671
Py_ssize_t refcount;

‎InternalDocs/frames.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,35 @@ The shim frame points to a special code object containing the `INTERPRETER_EXIT`
111111
instruction which cleans up the shim frame and returns.
112112

113113

114+
### Base frame
115+
116+
Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame"
117+
that serves as a sentinel at the bottom of the frame stack. This frame is allocated
118+
in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized
119+
when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`.
120+
121+
External profilers and sampling tools can validate that they have successfully unwound
122+
the complete call stack by checking that the frame chain terminates at the base frame.
123+
The `PyThreadState.base_frame` pointer provides the expected address to compare against.
124+
If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a
125+
race condition) and should be discarded.
126+
127+
The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because
128+
`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the
129+
public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access
130+
the address without needing internal headers.
131+
132+
See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c).
133+
134+
#### How profilers should use the base frame
135+
136+
External profilers should read `tstate->base_frame` before walking the stack, then
137+
walk from `tstate->current_frame` following `frame->previous` pointers until reaching
138+
a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the
139+
last frame address matches `base_frame`. If not, discard the sample as incomplete
140+
since the frame chain may have been in an inconsistent state due to concurrent updates.
141+
142+
114143
### Remote Profiling Frame Cache
115144

116145
The `last_profiled_frame` field in `PyThreadState` supports an optimization for

‎Lib/argparse.py‎

Lines changed: 94 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ def __init__(
189189
self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII)
190190
self._long_break_matcher = _re.compile(r'\n\n\n+')
191191

192+
self._set_color(False)
193+
192194
def _set_color(self, color):
193195
from _colorize import can_colorize, decolor, get_theme
194196

@@ -334,31 +336,15 @@ def _format_usage(self, usage, actions, groups, prefix):
334336
elif usage is None:
335337
prog = '%(prog)s' % dict(prog=self._prog)
336338

337-
# split optionals from positionals
338-
optionals = []
339-
positionals = []
340-
for action in actions:
341-
if action.option_strings:
342-
optionals.append(action)
343-
else:
344-
positionals.append(action)
345-
339+
parts, pos_start = self._get_actions_usage_parts(actions, groups)
346340
# build full usage string
347-
format = self._format_actions_usage
348-
action_usage = format(optionals + positionals, groups)
349-
usage = ' '.join([s for s in [prog, action_usage] if s])
341+
usage = ' '.join(filter(None, [prog, *parts]))
350342

351343
# wrap the usage parts if it's too long
352344
text_width = self._width - self._current_indent
353345
if len(prefix) + len(self._decolor(usage)) > text_width:
354346

355347
# break usage into wrappable parts
356-
# keep optionals and positionals together to preserve
357-
# mutually exclusive group formatting (gh-75949)
358-
all_actions = optionals + positionals
359-
parts, pos_start = self._get_actions_usage_parts_with_split(
360-
all_actions, groups, len(optionals)
361-
)
362348
opt_parts = parts[:pos_start]
363349
pos_parts = parts[pos_start:]
364350

@@ -417,125 +403,114 @@ def get_lines(parts, indent, prefix=None):
417403
# prefix with 'usage:'
418404
return f'{t.usage}{prefix}{t.reset}{usage}\n\n'
419405

420-
def _format_actions_usage(self, actions, groups):
421-
return ' '.join(self._get_actions_usage_parts(actions, groups))
422-
423406
def _is_long_option(self, string):
424407
return len(string) > 2
425408

426409
def _get_actions_usage_parts(self, actions, groups):
427-
parts, _ = self._get_actions_usage_parts_with_split(actions, groups)
428-
return parts
429-
430-
def _get_actions_usage_parts_with_split(self, actions, groups, opt_count=None):
431410
"""Get usage parts with split index for optionals/positionals.
432411
433412
Returns (parts, pos_start) where pos_start is the index in parts
434-
where positionals begin. When opt_count is None, pos_start is None.
413+
where positionals begin.
435414
This preserves mutually exclusive group formatting across the
436415
optionals/positionals boundary (gh-75949).
437416
"""
438-
# find group indices and identify actions in groups
439-
group_actions = set()
440-
inserts = {}
417+
actions = [action for action in actions if action.help is not SUPPRESS]
418+
# group actions by mutually exclusive groups
419+
action_groups = dict.fromkeys(actions)
441420
for group in groups:
442-
if not group._group_actions:
443-
raise ValueError(f'empty group {group}')
444-
445-
if all(action.help is SUPPRESS for action in group._group_actions):
446-
continue
447-
448-
try:
449-
start = min(actions.index(item) for item in group._group_actions)
450-
except ValueError:
451-
continue
452-
else:
453-
end = start + len(group._group_actions)
454-
if set(actions[start:end]) == set(group._group_actions):
455-
group_actions.update(group._group_actions)
456-
inserts[start, end] = group
421+
for action in group._group_actions:
422+
if action in action_groups:
423+
action_groups[action] = group
424+
# positional arguments keep their position
425+
positionals = []
426+
for action in actions:
427+
if not action.option_strings:
428+
group = action_groups.pop(action)
429+
if group:
430+
group_actions = [
431+
action2 for action2 in group._group_actions
432+
if action2.option_strings and
433+
action_groups.pop(action2, None)
434+
] + [action]
435+
positionals.append((group.required, group_actions))
436+
else:
437+
positionals.append((None, [action]))
438+
# the remaining optional arguments are sorted by the position of
439+
# the first option in the group
440+
optionals = []
441+
for action in actions:
442+
if action.option_strings and action in action_groups:
443+
group = action_groups.pop(action)
444+
if group:
445+
group_actions = [action] + [
446+
action2 for action2 in group._group_actions
447+
if action2.option_strings and
448+
action_groups.pop(action2, None)
449+
]
450+
optionals.append((group.required, group_actions))
451+
else:
452+
optionals.append((None, [action]))
457453

458454
# collect all actions format strings
459455
parts = []
460456
t = self._theme
461-
for action in actions:
462-
463-
# suppressed arguments are marked with None
464-
if action.help is SUPPRESS:
465-
part = None
466-
467-
# produce all arg strings
468-
elif not action.option_strings:
469-
default = self._get_default_metavar_for_positional(action)
470-
part = self._format_args(action, default)
471-
# if it's in a group, strip the outer []
472-
if action in group_actions:
473-
if part[0] == '[' and part[-1] == ']':
474-
part = part[1:-1]
475-
part = t.summary_action + part + t.reset
476-
477-
# produce the first way to invoke the option in brackets
478-
else:
479-
option_string = action.option_strings[0]
480-
if self._is_long_option(option_string):
481-
option_color = t.summary_long_option
457+
pos_start = None
458+
for i, (required, group) in enumerate(optionals + positionals):
459+
start = len(parts)
460+
if i == len(optionals):
461+
pos_start = start
462+
in_group = len(group) > 1
463+
for action in group:
464+
# produce all arg strings
465+
if not action.option_strings:
466+
default = self._get_default_metavar_for_positional(action)
467+
part = self._format_args(action, default)
468+
# if it's in a group, strip the outer []
469+
if in_group:
470+
if part[0] == '[' and part[-1] == ']':
471+
part = part[1:-1]
472+
part = t.summary_action + part + t.reset
473+
474+
# produce the first way to invoke the option in brackets
482475
else:
483-
option_color = t.summary_short_option
484-
485-
# if the Optional doesn't take a value, format is:
486-
# -s or --long
487-
if action.nargs == 0:
488-
part = action.format_usage()
489-
part = f"{option_color}{part}{t.reset}"
490-
491-
# if the Optional takes a value, format is:
492-
# -s ARGS or --long ARGS
493-
else:
494-
default = self._get_default_metavar_for_optional(action)
495-
args_string = self._format_args(action, default)
496-
part = (
497-
f"{option_color}{option_string} "
498-
f"{t.summary_label}{args_string}{t.reset}"
499-
)
500-
501-
# make it look optional if it's not required or in a group
502-
if not action.required and action not in group_actions:
503-
part = '[%s]' % part
476+
option_string = action.option_strings[0]
477+
if self._is_long_option(option_string):
478+
option_color = t.summary_long_option
479+
else:
480+
option_color = t.summary_short_option
504481

505-
# add the action string to the list
506-
parts.append(part)
482+
# if the Optional doesn't take a value, format is:
483+
# -s or --long
484+
if action.nargs == 0:
485+
part = action.format_usage()
486+
part = f"{option_color}{part}{t.reset}"
507487

508-
# group mutually exclusive actions
509-
inserted_separators_indices = set()
510-
for start, end in sorted(inserts, reverse=True):
511-
group = inserts[start, end]
512-
group_parts = [item for item in parts[start:end] if item is not None]
513-
group_size = len(group_parts)
514-
if group.required:
515-
open, close = "()" if group_size > 1 else ("", "")
516-
else:
517-
open, close = "[]"
518-
group_parts[0] = open + group_parts[0]
519-
group_parts[-1] = group_parts[-1] + close
520-
for i, part in enumerate(group_parts[:-1], start=start):
521-
# insert a separator if not already done in a nested group
522-
if i not in inserted_separators_indices:
523-
parts[i] = part + ' |'
524-
inserted_separators_indices.add(i)
525-
parts[start + group_size - 1] = group_parts[-1]
526-
for i in range(start + group_size, end):
527-
parts[i] = None
528-
529-
# if opt_count is provided, calculate where positionals start in
530-
# the final parts list (for wrapping onto separate lines).
531-
# Count before filtering None entries since indices shift after.
532-
if opt_count is not None:
533-
pos_start = sum(1 for p in parts[:opt_count] if p is not None)
534-
else:
535-
pos_start = None
536-
537-
# return the usage parts and split point (gh-75949)
538-
return [item for item in parts if item is not None], pos_start
488+
# if the Optional takes a value, format is:
489+
# -s ARGS or --long ARGS
490+
else:
491+
default = self._get_default_metavar_for_optional(action)
492+
args_string = self._format_args(action, default)
493+
part = (
494+
f"{option_color}{option_string} "
495+
f"{t.summary_label}{args_string}{t.reset}"
496+
)
497+
498+
# make it look optional if it's not required or in a group
499+
if not (action.required or required or in_group):
500+
part = '[%s]' % part
501+
502+
# add the action string to the list
503+
parts.append(part)
504+
505+
if in_group:
506+
parts[start] = ('(' if required else '[') + parts[start]
507+
for i in range(start, len(parts) - 1):
508+
parts[i] += ' |'
509+
parts[-1] += ')' if required else ']'
510+
511+
if pos_start is None:
512+
pos_start = len(parts)
513+
return parts, pos_start
539514

540515
def _format_text(self, text):
541516
if '%(prog)' in text:

0 commit comments

Comments
 (0)