10000 Thoroughly refactor the cases generator by gvanrossum · Pull Request #107151 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

Thoroughly refactor the cases generator #107151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 24, 2023
Merged

Thoroughly refactor the cases generator #107151

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
361dd47
Extract Formatter etc. into formatting.py; mv pars{er,ing}.py
gvanrossum Jul 22, 2023
4bc2c5c
Extract InstructionFlags etc. into flags.py
gvanrossum Jul 23, 2023
1710508
Extract Instruction etc. into instructions.py
gvanrossum Jul 23, 2023
4bf48d4
Extract Analyzer (but not Generator) into analysis.py
gvanrossum Jul 23, 2023
095d37f
Blackify what's left of generate_cases.py
gvanrossum Jul 23, 2023
2376916
Remove output filenames from Generator state
gvanrossum Jul 23, 2023
258bb41
Fix test_generated_cases.py
gvanrossum Jul 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Blackify what's left of generate_cases.py
  • Loading branch information
@gvanrossum
gvanrossum committed Jul 23, 2023
commit 095d37fe7e577add69c4a06760a5e92ea20dcee6
118 changes: 79 additions & 39 deletions Tools/cases_generator/generate_cases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

"""Generate the main interpreter switch.
Reads the instruction definitions from bytecodes.c.
Writes the cases to generated_cases.c.h, which is #included in ceval.c.
Expand Down Expand Up @@ -66,10 +65,18 @@
"-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
)
arg_parser.add_argument(
"-m", "--metadata", type=str, help="Generated C metadata", default=DEFAULT_METADATA_OUTPUT
"-m",
"--metadata",
type=str,
help="Generated C metadata",
default=DEFAULT_METADATA_OUTPUT,
)
arg_parser.add_argument(
"-p", "--pymetadata", type=str, help="Generated Python metadata", default=DEFAULT_PYMETADATA_OUTPUT
"-p",
"--pymetadata",
type=str,
help="Generated Python metadata",
default=DEFAULT_PYMETADATA_OUTPUT,
)
arg_parser.add_argument(
"-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
Expand Down Expand Up @@ -205,10 +212,14 @@ def write_function(
) -> None:
self.out.emit("")
self.out.emit("#ifndef NEED_OPCODE_METADATA")
self.out.emit(f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);")
self.out.emit(
f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);"
)
self.out.emit("#else")
self.out.emit("int")
self.out.emit(f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{")
self.out.emit(
f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{"
)
self.out.emit(" switch(opcode) {")
for instr, effect in data:
self.out.emit(f" case {instr.name}:")
Expand All @@ -229,7 +240,7 @@ def from_source_files(self) -> str:
try:
filename = os.path.relpath(filename, ROOT)
except ValueError:
# May happen on Windows if root and temp on different volumes
# May happen on Windows if root and temp on different volumes
pass
filenames.append(filename)
paths = f"\n{self.out.comment} ".join(filenames)
Expand Down Expand Up @@ -292,7 +303,8 @@ def write_metadata(self) -> None:
self.out.emit(
"#define IS_VALID_OPCODE(OP) \\\n"
" (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \\\n"
" (_PyOpcode_opcode_metadata[(OP)].valid_entry))")
" (_PyOpcode_opcode_metadata[(OP)].valid_entry))"
)

self.out.emit("")
InstructionFlags.emit_macros(self.out)
Expand All @@ -306,17 +318,23 @@ def write_metadata(self) -> None:

with self.out.block("struct opcode_macro_expansion", ";"):
self.out.emit("int nuops;")
self.out.emit("struct { int16_t uop; int8_t size; int8_t offset; } uops[8];")
self.out.emit(
"struct { int16_t uop; int8_t size; int8_t offset; } uops[8];"
)
self.out.emit("")

for key, value in OPARG_SIZES.items():
self.out.emit(f"#define {key} {value}")
self.out.emit("")

self.out.emit("#define OPCODE_METADATA_FMT(OP) "
"(_PyOpcode_opcode_metadata[(OP)].instr_format)")
self.out.emit(
"#define OPCODE_METADATA_FMT(OP) "
"(_PyOpcode_opcode_metadata[(OP)].instr_format)"
)
self.out.emit("#define SAME_OPCODE_METADATA(OP1, OP2) \\")
self.out.emit(" (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2))")
self.out.emit(
" (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2))"
)
self.out.emit("")

# Write metadata array declaration
Expand All @@ -325,15 +343,23 @@ def write_metadata(self) -> None:
self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256")
self.out.emit("")
self.out.emit("#ifndef NEED_OPCODE_METADATA")
self.out.emit("extern const struct opcode_metadata "
"_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];")
self.out.emit("extern const struct opcode_macro_expansion "
"_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];")
self.out.emit("extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];")
self.out.emit(
"extern const struct opcode_metadata "
"_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];"
)
self.out.emit(
"extern const struct opcode_macro_expansion "
"_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];"
)
self.out.emit(
"extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];"
)
self.out.emit("#else // if NEED_OPCODE_METADATA")

self.out.emit("const struct opcode_metadata "
"_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {")
self.out.emit(
"const struct opcode_metadata "
"_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {"
)

# Write metadata for each instruction
for thing in self.everything:
Expand Down Expand Up @@ -370,8 +396,12 @@ def write_metadata(self) -> None:
# Construct a dummy Component -- input/output mappings are not used
part = Component(instr, [], [], instr.active_caches)
self.write_macro_expansions(instr.name, [part])
elif instr.kind == "inst" and variable_used(instr.inst, "oparg1"):
assert variable_used(instr.inst, "oparg2"), "Half super-instr?"
elif instr.kind == "inst" and variable_used(
instr.inst, "oparg1"
):
assert variable_used(
instr.inst, "oparg2"
), "Half super-instr?"
self.write_super_expansions(instr.name)
case parsing.Macro():
mac = self.macro_instrs[thing.name]
Expand All @@ -381,41 +411,45 @@ def write_metadata(self) -> None:
case _:
typing.assert_never(thing)

with self.out.block("const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";"):
self.write_uop_items(lambda name, counter: f"[{name}] = \"{name}\",")
with self.out.block(
"const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";"
):
self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",')

self.out.emit("#endif // NEED_OPCODE_METADATA")

with open(self.pymetadata_filename, "w") as f:
# Create formatter
self.out = Formatter(f, 0, comment = "#")
self.out = Formatter(f, 0, comment="#")

self.write_provenance_header()

self.out.emit("")
self.out.emit("_specializations = {")
for name, family in self.families.items():
with self.out.indent():
self.out.emit(f"\"{family.name}\": [")
self.out.emit(f'"{family.name}": [')
with self.out.indent():
for m in family.members:
self.out.emit(f"\"{m}\",")
self.out.emit(f'"{m}",')
self.out.emit(f"],")
self.out.emit("}")

# Handle special case
self.out.emit("")
self.out.emit("# An irregular case:")
self.out.emit(
"_specializations[\"BINARY_OP\"].append("
"\"BINARY_OP_INPLACE_ADD_UNICODE\")")
'_specializations["BINARY_OP"].append('
'"BINARY_OP_INPLACE_ADD_UNICODE")'
)

# Make list of specialized instructions
self.out.emit("")
self.out.emit(
"_specialized_instructions = ["
"opcode for family in _specializations.values() for opcode in family"
"]")
"opcode for family in _specializations.values() for opcode in family"
"]"
)

def write_pseudo_instrs(self) -> None:
"""Write the IS_PSEUDO_INSTR macro"""
Expand Down Expand Up @@ -504,16 +538,18 @@ def write_super_expansions(self, name: str) -> None:
]
self.write_expansions(name, expansions)

def write_expansions(self, name: str, expansions: list[tuple[str, int, int]]) -> None:
pieces = [f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions]
def write_expansions(
self, name: str, expansions: list[tuple[str, int, int]]
) -> None:
pieces = [
f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions
]
self.out.emit(
f"[{name}] = "
f"{{ .nuops = {len(pieces)}, .uops = {{ {', '.join(pieces)} }} }},"
)

def emit_metadata_entry(
self, name: str, fmt: str, flags: InstructionFlags
) -> None:
def emit_metadata_entry(self, name: str, fmt: str, flags: InstructionFlags) -> None:
flag_names = flags.names(value=True)
if not flag_names:
flag_names.append("0")
Expand Down Expand Up @@ -600,11 +636,13 @@ def write_executor_instructions(self) -> None:
file=sys.stderr,
)

def write_overridden_instr_place_holder(self,
place_holder: OverriddenInstructionPlaceHolder) -> None:
def write_overridden_instr_place_holder(
self, place_holder: OverriddenInstructionPlaceHolder
) -> None:
self.out.emit("")
self.out.emit(
f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition")
f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition"
)

def write_instr(self, instr: Instruction) -> None:
name = instr.name
Expand Down Expand Up @@ -675,7 +713,7 @@ def wrap_macro(self, mac: MacroInstruction):

yield

self.out.stack_adjust(ieffects[:mac.initial_sp], mac.stack[:mac.final_sp])
self.out.stack_adjust(ieffects[: mac.initial_sp], mac.stack[: mac.final_sp])

for i, var in enumerate(reversed(mac.stack[: mac.final_sp]), 1):
dst = StackEffect(f"stack_pointer[-{i}]", "")
Expand All @@ -691,7 +729,9 @@ def main():
args.input.append(DEFAULT_INPUT)

# Raises OSError if input unreadable
a = Generator(args.input, args.output, args.metadata, args.pymetadata, args.executor_cases)
a = Generator(
args.input, args.output, args.metadata, args.pymetadata, args.executor_cases
)

if args.emit_line_directives:
a.emit_line_directives = True
Expand Down
0