Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit aefdebd

Browse files
markshannonmarkshannon
authored
GH-111485: Factor out opcode ID generator from the main cases generator. (GH-112831)
1 parent 15a80b1 commit aefdebd

File tree

7 files changed

+203
-72
lines changed

7 files changed

+203
-72
lines changed

‎Include/opcode_ids.h

Lines changed: 5 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Makefile.pre.in

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1587,13 +1587,14 @@ regen-cases:
15871587
$(PYTHON_FOR_REGEN) \
15881588
$(srcdir)/Tools/cases_generator/generate_cases.py \
15891589
$(CASESFLAG) \
1590-
-n $(srcdir)/Include/opcode_ids.h.new \
15911590
-t $(srcdir)/Python/opcode_targets.h.new \
15921591
-m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \
15931592
-e $(srcdir)/Python/executor_cases.c.h.new \
15941593
-p $(srcdir)/Lib/_opcode_metadata.py.new \
15951594
-a $(srcdir)/Python/abstract_interp_cases.c.h.new \
15961595
$(srcdir)/Python/bytecodes.c
1596+
$(PYTHON_FOR_REGEN) \
1597+
$(srcdir)/Tools/cases_generator/opcode_id_generator.py -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c
15971598
$(PYTHON_FOR_REGEN) \
15981599
$(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
15991600
$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new

‎Tools/cases_generator/cwriter.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,13 @@ def maybe_indent(self, txt: str) -> None:
4848
if offset <= self.indents[-1] or offset > 40:
4949
offset = self.indents[-1] + 4
5050
self.indents.append(offset)
51-
elif "{" in txt or is_label(txt):
51+
if is_label(txt):
5252
self.indents.append(self.indents[-1] + 4)
53+
elif "{" in txt:
54+
if 'extern "C"' in txt:
55+
self.indents.append(self.indents[-1])
56+
else:
57+
self.indents.append(self.indents[-1] + 4)
5358

5459
def emit_text(self, txt: str) -> None:
5560
self.out.write(txt)

‎Tools/cases_generator/generate_cases.py

Lines changed: 3 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,6 @@
101101
arg_parser.add_argument(
102102
"-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
103103
)
104-
arg_parser.add_argument(
105-
"-n",
106-
"--opcode_ids_h",
107-
type=str,
108-
help="Header file with opcode number definitions",
109-
default=DEFAULT_OPCODE_IDS_H_OUTPUT,
110-
)
111104
arg_parser.add_argument(
112105
"-t",
113106
"--opcode_targets_h",
@@ -334,42 +327,8 @@ def map_op(op: int, name: str) -> None:
334327
self.opmap = opmap
335328
self.markers = markers
336329

337-
def write_opcode_ids(
338-
self, opcode_ids_h_filename: str, opcode_targets_filename: str
339-
) -> None:
340-
"""Write header file that defined the opcode IDs"""
341-
342-
with open(opcode_ids_h_filename, "w") as f:
343-
# Create formatter
344-
self.out = Formatter(f, 0)
345-
346-
self.write_provenance_header()
347-
348-
self.out.emit("")
349-
self.out.emit("#ifndef Py_OPCODE_IDS_H")
350-
self.out.emit("#define Py_OPCODE_IDS_H")
351-
self.out.emit("#ifdef __cplusplus")
352-
self.out.emit('extern "C" {')
353-
self.out.emit("#endif")
354-
self.out.emit("")
355-
self.out.emit("/* Instruction opcodes for compiled code */")
356-
357-
def define(name: str, opcode: int) -> None:
358-
self.out.emit(f"#define {name:<38} {opcode:>3}")
359-
360-
all_pairs: list[tuple[int, int, str]] = []
361-
# the second item in the tuple sorts the markers before the ops
362-
all_pairs.extend((i, 1, name) for (name, i) in self.markers.items())
363-
all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items())
364-
for i, _, name in sorted(all_pairs):
365-
assert name is not None
366-
define(name, i)
367-
368-
self.out.emit("")
369-
self.out.emit("#ifdef __cplusplus")
370-
self.out.emit("}")
371-
self.out.emit("#endif")
372-
self.out.emit("#endif /* !Py_OPCODE_IDS_H */")
330+
def write_opcode_targets(self, opcode_targets_filename: str) -> None:
331+
"""Write header file that defines the jump target table"""
373332

374333
with open(opcode_targets_filename, "w") as f:
375334
# Create formatter
@@ -885,7 +844,7 @@ def main() -> None:
885844
# These raise OSError if output can't be written
886845

887846
a.assign_opcode_ids()
888-
a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h)
847+
a.write_opcode_targets(args.opcode_targets_h)
889848
a.write_metadata(args.metadata, args.pymetadata)
890849
a.write_executor_instructions(args.executor_cases, args.emit_line_directives)
891850
a.write_abstract_interpreter_instructions(

‎Tools/cases_generator/generators_common.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from pathlib import Path
2+
from typing import TextIO
3+
4+
ROOT = Path(__file__).parent.parent.parent
5+
DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute()
6+
7+
8+
def root_relative_path(filename: str) -> str:
9+
return Path(filename).relative_to(ROOT).as_posix()
10+
11+
12+
def write_header(generator: str, source: str, outfile: TextIO) -> None:
13+
outfile.write(
14+
f"""// This file is generated by {root_relative_path(generator)}
15+
// from:
16+
// {source}
17+
// Do not edit!
18+
"""
19+
)

‎Tools/cases_generator/opcode_id_generator.py

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
"""Generate the list of opcode IDs.
2+
Reads the instruction definitions from bytecodes.c.
3+
Writes the IDs to opcode._ids.h by default.
4+
"""
5+
6+
import argparse
7+
import os.path
8+
import sys
9+
10+
from analyzer import (
11+
Analysis,
12+
Instruction,
13+
analyze_files,
14+
)
15+
from generators_common import (
16+
DEFAULT_INPUT,
17+
ROOT,
18+
write_header,
19+
)
20+
from cwriter import CWriter
21+
from typing import TextIO
22+
23+
24+
DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h"
25+
26+
27+
def generate_opcode_header(filenames: str, analysis: Analysis, outfile: TextIO) -> None:
28+
write_header(__file__, filenames, outfile)
29+
out = CWriter(outfile, 0, False)
30+
out.emit("\n")
31+
instmap: dict[str, int] = {}
32+
33+
# 0 is reserved for cache entries. This helps debugging.
34+
instmap["CACHE"] = 0
35+
36+
# 17 is reserved as it is the initial value for the specializing counter.
37+
# This helps catch cases where we attempt to execute a cache.
38+
instmap["RESERVED"] = 17
39+
40+
# 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py
41+
instmap["RESUME"] = 149
42+
instmap["INSTRUMENTED_LINE"] = 254
43+
44+
instrumented = [
45+
name for name in analysis.instructions if name.startswith("INSTRUMENTED")
46+
]
47+
48+
# Special case: this instruction is implemented in ceval.c
49+
# rather than bytecodes.c, so we need to add it explicitly
50+
# here (at least until we add something to bytecodes.c to
51+
# declare external instructions).
52+
instrumented.append("INSTRUMENTED_LINE")
53+
54+
specialized: set[str] = set()
55+
no_arg: list[str] = []
56+
has_arg: list[str] = []
57+
58+
for family in analysis.families.values():
59+
specialized.update(inst.name for inst in family.members)
60+
61+
for inst in analysis.instructions.values():
62+
name = inst.name
63+
if name in specialized:
64+
continue
65+
if name in instrumented:
66+
continue
67+
if inst.properties.oparg:
68+
has_arg.append(name)
69+
else:
70+
no_arg.append(name)
71+
72+
# Specialized ops appear in their own section
73+
# Instrumented opcodes are at the end of the valid range
74+
min_internal = 150
75+
min_instrumented = 254 - (len(instrumented) - 1)
76+
assert min_internal + len(specialized) < min_instrumented
77+
78+
next_opcode = 1
79+
80+
def add_instruction(name: str) -> None:
81+
nonlocal next_opcode
82+
if name in instmap:
83+
return # Pre-defined name
84+
while next_opcode in instmap.values():
85+
next_opcode += 1
86+
instmap[name] = next_opcode
87+
next_opcode += 1
88+
89+
for name in sorted(no_arg):
90+
add_instruction(name)
91+
for name in sorted(has_arg):
92+
add_instruction(name)
93+
# For compatibility
94+
next_opcode = min_internal
95+
for name in sorted(specialized):
96+
add_instruction(name)
97+
next_opcode = min_instrumented
98+
for name in instrumented:
99+
add_instruction(name)
100+
101+
for op, name in enumerate(sorted(analysis.pseudos), 256):
102+
instmap[name] = op
103+
104+
assert 255 not in instmap.values()
105+
106+
out.emit(
107+
"""#ifndef Py_OPCODE_IDS_H
108+
#define Py_OPCODE_IDS_H
109+
#ifdef __cplusplus
110+
extern "C" {
111+
#endif
112+
113+
/* Instruction opcodes for compiled code */
114+
"""
115+
)
116+
117+
def write_define(name: str, op: int) -> None:
118+
out.emit(f"#define {name:<38} {op:>3}\n")
119+
120+
for op, name in sorted([(op, name) for (name, op) in instmap.items()]):
121+
write_define(name, op)
122+
123+
out.emit("\n")
124+
write_define("HAVE_ARGUMENT", len(no_arg))
125+
write_define("MIN_INSTRUMENTED_OPCODE", min_instrumented)
126+
127+
out.emit("\n")
128+
out.emit("#ifdef __cplusplus\n")
129+
out.emit("}\n")
130+
out.emit("#endif\n")
131+
out.emit("#endif /* !Py_OPCODE_IDS_H */\n")
132+
133+
134+
arg_parser = argparse.ArgumentParser(
135+
description="Generate the header file with all opcode IDs.",
136+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
137+
)
138+
139+
arg_parser.add_argument(
140+
"-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
141+
)
142+
143+
arg_parser.add_argument(
144+
"input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
145+
)
146+
147+
if __name__ == "__main__":
148+
args = arg_parser.parse_args()
149+
if len(args.input) == 0:
150+
args.input.append(DEFAULT_INPUT)
151+
data = analyze_files(args.input)
152+
with open(args.output, "w") as outfile:
153+
generate_opcode_header(args.input, data, outfile)

‎Tools/cases_generator/tier1_generator.py

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,18 @@
1717
StackItem,
1818
analysis_error,
1919
)
20+
from generators_common import (
21+
DEFAULT_INPUT,
22+
ROOT,
23+
write_header,
24+
)
2025
from cwriter import CWriter
2126
from typing import TextIO, Iterator
2227
from lexer import Token
2328
from stack import StackOffset
2429

2530

26-
HERE = os.path.dirname(__file__)
27-
ROOT = os.path.join(HERE, "../..")
28-
THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/")
29-
30-
DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c"))
31-
DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h"))
32-
33-
34-
def write_header(filename: str, outfile: TextIO) -> None:
35-
outfile.write(
36-
f"""// This file is generated by {THIS}
37-
// from:
38-
// {filename}
39-
// Do not edit!
40-
41-
#ifdef TIER_TWO
42-
#error "This file is for Tier 1 only"
43-
#endif
44-
#define TIER_ONE 1
45-
"""
46-
)
31+
DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h"
4732

4833

4934
FOOTER = "#undef TIER_ONE\n"
@@ -351,7 +336,15 @@ def uses_this(inst: Instruction) -> bool:
351336
def generate_tier1(
352337
filenames: str, analysis: Analysis, outfile: TextIO, lines: bool
353338
) -> None:
354-
write_header(filenames, outfile)
339+
write_header(__file__, filenames, outfile)
340+
outfile.write(
341+
"""
342+
#ifdef TIER_TWO
343+
#error "This file is for Tier 1 only"
344+
#endif
345+
#define TIER_ONE 1
346+
"""
347+
)
355348
out = CWriter(outfile, 2, lines)
356349
out.emit("\n")
357350
for name, inst in sorted(analysis.instructions.items()):

0 commit comments

Comments
 (0)