pythonGH-111485: Factor out opcode ID generator from the main cases g…

…enerator. (pythonGH-112831)

Makefile.pre.in

@@ -1587,13 +1587,14 @@ regen-cases:
  $(PYTHON_FOR_REGEN) \
  $(srcdir)/Tools/cases_generator/generate_cases.py \
  $(CASESFLAG) \
- -n $(srcdir)/Include/opcode_ids.h.new \
  -t $(srcdir)/Python/opcode_targets.h.new \
  -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \
  -e $(srcdir)/Python/executor_cases.c.h.new \
  -p $(srcdir)/Lib/_opcode_metadata.py.new \
  -a $(srcdir)/Python/abstract_interp_cases.c.h.new \
  $(srcdir)/Python/bytecodes.c
+ $(PYTHON_FOR_REGEN) \
+ $(srcdir)/Tools/cases_generator/opcode_id_generator.py -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c
  $(PYTHON_FOR_REGEN) \
  $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
  $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new

Tools/cases_generator/cwriter.py

@@ -48,8 +48,13 @@ def maybe_indent(self, txt: str) -> None:
  if offset <= self.indents[-1] or offset > 40:
  offset = self.indents[-1] + 4
  self.indents.append(offset)
- elif "{" in txt or is_label(txt):
+ if is_label(txt):
  self.indents.append(self.indents[-1] + 4)
+ elif "{" in txt:
+ if 'extern "C"' in txt:
+ self.indents.append(self.indents[-1])
+ else:
+ self.indents.append(self.indents[-1] + 4)
 
  def emit_text(self, txt: str) -> None:
  self.out.write(txt)

Tools/cases_generator/generate_cases.py

@@ -101,13 +101,6 @@
 arg_parser.add_argument(
  "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
 )
-arg_parser.add_argument(
- "-n",
- "--opcode_ids_h",
- type=str,
- help="Header file with opcode number definitions",
- default=DEFAULT_OPCODE_IDS_H_OUTPUT,
-)
 arg_parser.add_argument(
  "-t",
  "--opcode_targets_h",
@@ -334,42 +327,8 @@ def map_op(op: int, name: str) -> None:
  self.opmap = opmap
  self.markers = markers
 
- def write_opcode_ids(
- self, opcode_ids_h_filename: str, opcode_targets_filename: str
- ) -> None:
- """Write header file that defined the opcode IDs"""
-
- with open(opcode_ids_h_filename, "w") as f:
- # Create formatter
- self.out = Formatter(f, 0)
-
- self.write_provenance_header()
-
- self.out.emit("")
- self.out.emit("#ifndef Py_OPCODE_IDS_H")
- self.out.emit("#define Py_OPCODE_IDS_H")
- self.out.emit("#ifdef __cplusplus")
- self.out.emit('extern "C" {')
- self.out.emit("#endif")
- self.out.emit("")
- self.out.emit("/* Instruction opcodes for compiled code */")
-
- def define(name: str, opcode: int) -> None:
- self.out.emit(f"#define {name:<38} {opcode:>3}")
-
- all_pairs: list[tuple[int, int, str]] = []
- # the second item in the tuple sorts the markers before the ops
- all_pairs.extend((i, 1, name) for (name, i) in self.markers.items())
- all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items())
- for i, _, name in sorted(all_pairs):
- assert name is not None
- define(name, i)
-
- self.out.emit("")
- self.out.emit("#ifdef __cplusplus")
- self.out.emit("}")
- self.out.emit("#endif")
- self.out.emit("#endif /* !Py_OPCODE_IDS_H */")
+ def write_opcode_targets(self, opcode_targets_filename: str) -> None:
+ """Write header file that defines the jump target table"""
 
  with open(opcode_targets_filename, "w") as f:
  # Create formatter
@@ -885,7 +844,7 @@ def main() -> None:
  # These raise OSError if output can't be written
 
  a.assign_opcode_ids()
- a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h)
+ a.write_opcode_targets(args.opcode_targets_h)
  a.write_metadata(args.metadata, args.pymetadata)
  a.write_executor_instructions(args.executor_cases, args.emit_line_directives)
  a.write_abstract_interpreter_instructions(

Tools/cases_generator/generators_common.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+from typing import TextIO
+
+ROOT = Path(__file__).parent.parent.parent
+DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute()
+
+
+def root_relative_path(filename: str) -> str:
+ return Path(filename).relative_to(ROOT).as_posix()
+
+
+def write_header(generator: str, source: str, outfile: TextIO) -> None:
+ outfile.write(
+ f"""// This file is generated by {root_relative_path(generator)}
+// from:
+// {source}
+// Do not edit!
+"""
+ )

Tools/cases_generator/opcode_id_generator.py

@@ -0,0 +1,153 @@
+"""Generate the list of opcode IDs.
+Reads the instruction definitions from bytecodes.c.
+Writes the IDs to opcode._ids.h by default.
+"""
+
+import argparse
+import os.path
+import sys
+
+from analyzer import (
+ Analysis,
+ Instruction,
+ analyze_files,
+)
+from generators_common import (
+ DEFAULT_INPUT,
+ ROOT,
+ write_header,
+)
+from cwriter import CWriter
+from typing import TextIO
+
+
+DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h"
+
+
+def generate_opcode_header(filenames: str, analysis: Analysis, outfile: TextIO) -> None:
+ write_header(__file__, filenames, outfile)
+ out = CWriter(outfile, 0, False)
+ out.emit("\n")
+ instmap: dict[str, int] = {}
+
+ # 0 is reserved for cache entries. This helps debugging.
+ instmap["CACHE"] = 0
+
+ # 17 is reserved as it is the initial value for the specializing counter.
+ # This helps catch cases where we attempt to execute a cache.
+ instmap["RESERVED"] = 17
+
+ # 149 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py
+ instmap["RESUME"] = 149
+ instmap["INSTRUMENTED_LINE"] = 254
+
+ instrumented = [
+ name for name in analysis.instructions if name.startswith("INSTRUMENTED")
+ ]
+
+ # Special case: this instruction is implemented in ceval.c
+ # rather than bytecodes.c, so we need to add it explicitly
+ # here (at least until we add something to bytecodes.c to
+ # declare external instructions).
+ instrumented.append("INSTRUMENTED_LINE")
+
+ specialized: set[str] = set()
+ no_arg: list[str] = []
+ has_arg: list[str] = []
+
+ for family in analysis.families.values():
+ specialized.update(inst.name for inst in family.members)
+
+ for inst in analysis.instructions.values():
+ name = inst.name
+ if name in specialized:
+ continue
+ if name in instrumented:
+ continue
+ if inst.properties.oparg:
+ has_arg.append(name)
+ else:
+ no_arg.append(name)
+
+ # Specialized ops appear in their own section
+ # Instrumented opcodes are at the end of the valid range
+ min_internal = 150
+ min_instrumented = 254 - (len(instrumented) - 1)
+ assert min_internal + len(specialized) < min_instrumented
+
+ next_opcode = 1
+
+ def add_instruction(name: str) -> None:
+ nonlocal next_opcode
+ if name in instmap:
+ return # Pre-defined name
+ while next_opcode in instmap.values():
+ next_opcode += 1
+ instmap[name] = next_opcode
+ next_opcode += 1
+
+ for name in sorted(no_arg):
+ add_instruction(name)
+ for name in sorted(has_arg):
+ add_instruction(name)
+ # For compatibility
+ next_opcode = min_internal
+ for name in sorted(specialized):
+ add_instruction(name)
+ next_opcode = min_instrumented
+ for name in instrumented:
+ add_instruction(name)
+
+ for op, name in enumerate(sorted(analysis.pseudos), 256):
+ instmap[name] = op
+
+ assert 255 not in instmap.values()
+
+ out.emit(
+ """#ifndef Py_OPCODE_IDS_H
+#define Py_OPCODE_IDS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Instruction opcodes for compiled code */
+"""
+ )
+
+ def write_define(name: str, op: int) -> None:
+ out.emit(f"#define {name:<38} {op:>3}\n")
+
+ for op, name in sorted([(op, name) for (name, op) in instmap.items()]):
+ write_define(name, op)
+
+ out.emit("\n")
+ write_define("HAVE_ARGUMENT", len(no_arg))
+ write_define("MIN_INSTRUMENTED_OPCODE", min_instrumented)
+
+ out.emit("\n")
+ out.emit("#ifdef __cplusplus\n")
+ out.emit("}\n")
+ out.emit("#endif\n")
+ out.emit("#endif /* !Py_OPCODE_IDS_H */\n")
+
+
+arg_parser = argparse.ArgumentParser(
+ description="Generate the header file with all opcode IDs.",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+)
+
+arg_parser.add_argument(
+ "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
+)
+
+arg_parser.add_argument(
+ "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
+)
+
+if __name__ == "__main__":
+ args = arg_parser.parse_args()
+ if len(args.input) == 0:
+ args.input.append(DEFAULT_INPUT)
+ data = analyze_files(args.input)
+ with open(args.output, "w") as outfile:
+ generate_opcode_header(args.input, data, outfile)

Tools/cases_generator/tier1_generator.py

@@ -17,33 +17,18 @@
  StackItem,
  analysis_error,
 )
+from generators_common import (
+ DEFAULT_INPUT,
+ ROOT,
+ write_header,
+)
 from cwriter import CWriter
 from typing import TextIO, Iterator
 from lexer import Token
 from stack import StackOffset
 
 
-HERE = os.path.dirname(__file__)
-ROOT = os.path.join(HERE, "../..")
-THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/")
-
-DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c"))
-DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h"))
-
-
-def write_header(filename: str, outfile: TextIO) -> None:
- outfile.write(
- f"""// This file is generated by {THIS}
-// from:
-// {filename}
-// Do not edit!
-
-#ifdef TIER_TWO
- #error "This file is for Tier 1 only"
-#endif
-#define TIER_ONE 1
-"""
- )
+DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h"
 
 
 FOOTER = "#undef TIER_ONE\n"
@@ -351,7 +336,15 @@ def uses_this(inst: Instruction) -> bool:
 def generate_tier1(
  filenames: str, analysis: Analysis, outfile: TextIO, lines: bool
 ) -> None:
- write_header(filenames, outfile)
+ write_header(__file__, filenames, outfile)
+ outfile.write(
+ """
+#ifdef TIER_TWO
+ #error "This file is for Tier 1 only"
+#endif
+#define TIER_ONE 1
+"""
+ )
  out = CWriter(outfile, 2, lines)
  out.emit("\n")
  for name, inst in sorted(analysis.instructions.items()):