bpo-45873: Get rid of bootstrap_python (python#29717)

Instead we use $(PYTHON_FOR_REGEN) .../deepfreeze.py with the
frozen .h file as input, as we did for Windows in bpo-45850.

We also get rid of the code that generates the .h files
when make regen-frozen is run (i.e., .../make_frozen.py),
and the MANIFEST file.

Restore Python 3.8 and 3.9 as Windows host Python again

Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com>

Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst

@@ -0,0 +1 @@
+Get rid of the ``_bootstrap_python`` build step. The deepfreeze.py script is now run using ``$(PYTHON_FOR_REGEN)`` which can be Python 3.7 or newer (on Windows, 3.8 or newer).

PCbuild/find_python.bat

@@ -31,13 +31,13 @@
 @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
 
 @rem If we have Python in externals, use that one
-@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
 
 @rem If HOST_PYTHON is recent enough, use that
-@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
 
 @rem If py.exe finds a recent enough version, use that one
-@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
+@for %%p in (3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
 
 @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
 @set _Py_NUGET=%NUGET%

Tools/scripts/deepfreeze.py

@@ -7,7 +7,7 @@
 import re
 import time
 import types
-import typing
+from typing import Dict, FrozenSet, Tuple, TextIO
 
 import umarshal
 
@@ -42,13 +42,14 @@ def get_localsplus(code: types.CodeType):
 
 
 def get_localsplus_counts(code: types.CodeType,
- names: tuple[str, ...],
- kinds: bytes) -> tuple[int, int, int, int]:
+ names: Tuple[str, ...],
+ kinds: bytes) -> Tuple[int, int, int, int]:
  nlocals = 0
  nplaincellvars = 0
  ncellvars = 0
  nfreevars = 0
- for name, kind in zip(names, kinds, strict=True):
+ assert len(names) == len(kinds)
+ for name, kind in zip(names, kinds):
  if kind & CO_FAST_LOCAL:
  nlocals += 1
  if kind & CO_FAST_CELL:
@@ -71,7 +72,7 @@ def get_localsplus_counts(code: types.CodeType,
 PyUnicode_4BYTE_KIND = 4
 
 
-def analyze_character_width(s: str) -> tuple[int, bool]:
+def analyze_character_width(s: str) -> Tuple[int, bool]:
  maxchar = ' '
  for c in s:
  maxchar = max(maxchar, c)
@@ -86,12 +87,17 @@ def analyze_character_width(s: str) -> tuple[int, bool]:
  return kind, ascii
 
 
+def removesuffix(base: str, suffix: str) -> str:
+ if base.endswith(suffix):
+ return base[:len(base) - len(suffix)]
+ return base
+
 class Printer:
 
- def __init__(self, file: typing.TextIO):
+ def __init__(self, file: TextIO):
  self.level = 0
  self.file = file
- self.cache: dict[tuple[type, object], str] = {}
+ self.cache: Dict[Tuple[type, object], str] = {}
  self.hits, self.misses = 0, 0
  self.patchups: list[str] = []
  self.write('#include "Python.h"')
@@ -231,7 +237,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
  # otherwise MSVC doesn't like it.
  self.write(f".co_consts = {co_consts},")
  self.write(f".co_names = {co_names},")
- self.write(f".co_firstinstr = (_Py_CODEUNIT *) {co_code.removesuffix('.ob_base.ob_base')}.ob_sval,")
+ self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,")
  self.write(f".co_exceptiontable = {co_exceptiontable},")
  self.field(code, "co_flags")
  self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
@@ -259,7 +265,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
  self.write(f".co_freevars = {co_freevars},")
  return f"& {name}.ob_base"
 
- def generate_tuple(self, name: str, t: tuple[object, ...]) -> str:
+ def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
  items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)]
  self.write("static")
  with self.indent():
@@ -323,7 +329,7 @@ def generate_complex(self, name: str, z: complex) -> str:
  self.write(f".cval = {{ {z.real}, {z.imag} }},")
  return f"&{name}.ob_base"
 
- def generate_frozenset(self, name: str, fs: frozenset[object]) -> str:
+ def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str:
  ret = self.generate_tuple(name, tuple(sorted(fs)))
  self.write("// TODO: The above tuple should be a frozenset")
  return ret
@@ -336,34 +342,33 @@ def generate(self, name: str, obj: object) -> str:
  # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}")
  return self.cache[key]
  self.misses += 1
- match obj:
- case types.CodeType() | umarshal.Code() as code:
- val = self.generate_code(name, code)
- case tuple(t):
- val = self.generate_tuple(name, t)
- case str(s):
- val = self.generate_unicode(name, s)
- case bytes(b):
- val = self.generate_bytes(name, b)
- case True:
- return "Py_True"
- case False:
- return "Py_False"
- case int(i):
- val = self.generate_int(name, i)
- case float(x):
- val = self.generate_float(name, x)
- case complex() as z:
- val = self.generate_complex(name, z)
- case frozenset(fs):
- val = self.generate_frozenset(name, fs)
- case builtins.Ellipsis:
- return "Py_Ellipsis"
- case None:
- return "Py_None"
- case _:
- raise TypeError(
- f"Cannot generate code for {type(obj).__name__} object")
+ if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code):
+ val = self.generate_code(name, obj)
+ elif isinstance(obj, tuple):
+ val = self.generate_tuple(name, obj)
+ elif isinstance(obj, str):
+ val = self.generate_unicode(name, obj)
+ elif isinstance(obj, bytes):
+ val = self.generate_bytes(name, obj)
+ elif obj is True:
+ return "Py_True"
+ elif obj is False:
+ return "Py_False"
+ elif isinstance(obj, int):
+ val = self.generate_int(name, obj)
+ elif isinstance(obj, float):
+ val = self.generate_float(name, obj)
+ elif isinstance(obj, complex):
+ val = self.generate_complex(name, obj)
+ elif isinstance(obj, frozenset):
+ val = self.generate_frozenset(name, obj)
+ elif obj is builtins.Ellipsis:
+ return "Py_Ellipsis"
+ elif obj is None:
+ return "Py_None"
+ else:
+ raise TypeError(
+ f"Cannot generate code for {type(obj).__name__} object")
  # print(f"Cache store {key!r:.40}: {val!r:.40}")
  self.cache[key] = val
  return val
@@ -393,12 +398,12 @@ def decode_frozen_data(source: str) -> types.CodeType:
  del lines[0]
  while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None:
  del lines[-1]
- values: tuple[int, ...] = ast.literal_eval("".join(lines))
+ values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip())
  data = bytes(values)
  return umarshal.loads(data)
 
 
-def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None:
+def generate(source: str, filename: str, modname: str, file: TextIO) -> None:
  if is_frozen_header(source):
  code = decode_frozen_data(source)
  else:
@@ -439,7 +444,7 @@ def main() -> None:
  verbose = args.verbose
  with open(args.file, encoding="utf-8") as f:
  source = f.read()
- modname = args.module or os.path.basename(args.file).removesuffix(".py")
+ modname = args.module or removesuffix(os.path.basename(args.file), ".py")
  output = args.output or modname + ".c"
  with open(output, "w", encoding="utf-8") as file:
  with report_time("generate"):

Tools/scripts/freeze_modules.py

@@ -25,31 +25,6 @@
 # need to be updated.
 MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
 
-if sys.platform != "win32":
- TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
- if not os.path.isfile(TOOL):
- # When building out of the source tree, get the tool from directory
- # of the Python executable
- TOOL = os.path.dirname(sys.executable)
- TOOL = os.path.join(TOOL, 'Programs', '_freeze_module')
- TOOL = os.path.abspath(TOOL)
- if not os.path.isfile(TOOL):
- sys.exit("ERROR: missing _freeze_module")
-else:
- def find_tool():
- archs = ['amd64', 'win32']
- if platform.machine() == "ARM64":
- archs.append('arm64')
- for arch in archs:
- for exe in ['_freeze_module.exe', '_freeze_module_d.exe']:
- tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe)
- if os.path.isfile(tool):
- return tool
- sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat")
- TOOL = find_tool()
- del find_tool
-
-MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST')
 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
 PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
@@ -480,45 +455,6 @@ def replace_block(lines, start_marker, end_marker, replacements, file):
  return lines[:start_pos + 1] + replacements + lines[end_pos:]
 
 
-def regen_manifest(modules):
- header = 'module ispkg source frozen checksum'.split()
- widths = [5] * len(header)
- rows = []
- for mod in modules:
- info = mod.summarize()
- row = []
- for i, col in enumerate(header):
- value = info[col]
- if col == 'checksum':
- value = value[:12]
- elif col == 'ispkg':
- value = 'YES' if value else 'no'
- widths[i] = max(widths[i], len(value))
- row.append(value or '-')
- rows.append(row)
-
- modlines = [
- '# The list of frozen modules with key information.',
- '# Note that the "check_generated_files" CI job will identify',
- '# when source files were changed but regen-frozen wasn\'t run.',
- '# This file is auto-generated by Tools/scripts/freeze_modules.py.',
- ' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(),
- ' '.join('-' * w for w in widths),
- ]
- for row in rows:
- for i, w in enumerate(widths):
- if header[i] == 'ispkg':
- row[i] = row[i].center(w)
- else:
- row[i] = row[i].ljust(w)
- modlines.append(' '.join(row).rstrip())
-
- print(f'# Updating {os.path.relpath(MANIFEST)}')
- with open(MANIFEST, 'w', encoding="utf-8") as outfile:
- lines = (l + '\n' for l in modlines)
- outfile.writelines(lines)
-
-
 def regen_frozen(modules):
  headerlines = []
  parentdir = os.path.dirname(FROZEN_FILE)
@@ -648,11 +584,11 @@ def regen_makefile(modules):
  deepfreezefiles.append(f"\t\t{ofile} \\")
 
  # Also add a deepfreeze rule.
- deepfreezerules.append(f'{cfile}: $(srcdir)/{_pyfile} $(DEEPFREEZE_DEPS)')
- deepfreezerules.append(f'\t@echo "Deepfreezing {cfile} from {_pyfile}"')
- deepfreezerules.append(f"\t@./$(BOOTSTRAP) \\")
- deepfreezerules.append(f"\t\t$(srcdir)/Tools/scripts/deepfreeze.py \\")
- deepfreezerules.append(f"\t\t$(srcdir)/{_pyfile} -m {src.frozenid} -o {cfile}")
+ deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)')
+ deepfreezerules.append(
+  f"\t$(PYTHON_FOR_REGEN) "
+  f"$(srcdir)/Tools/scripts/deepfreeze.py "
+  f"{header} -m {src.frozenid} -o {cfile}")
  deepfreezerules.append('')
 
  for src in _iter_sources(modules):
@@ -663,7 +599,7 @@ def regen_makefile(modules):
  pyfiles.append(f'\t\t{pyfile} \\')
 
  freeze = (f'Programs/_freeze_module {src.frozenid} '
- f'$(srcdir)/{pyfile} $(srcdir)/{header}')
+ f'$(srcdir)/{pyfile} {header}')
  rules.extend([
  f'{header}: Programs/_freeze_module {pyfile}',
  f'\t{freeze}',
@@ -774,32 +710,6 @@ def regen_pcbuild(modules):
  outfile.writelines(lines)
 
 
-#######################################
-# freezing modules
-
-def freeze_module(modname, pyfile=None, destdir=MODULES_DIR):
- """Generate the frozen module .h file for the given module."""
- tmpsuffix = f'.{int(time.time())}'
- for modname, pyfile, ispkg in resolve_modules(modname, pyfile):
- frozenfile = resolve_frozen_file(modname, destdir)
- _freeze_module(modname, pyfile, frozenfile, tmpsuffix)
-
-
-def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix):
- tmpfile = f'{frozenfile}.{int(time.time())}'
-
- argv = [TOOL, frozenid, pyfile, tmpfile]
- print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True)
- try:
- subprocess.run(argv, check=True)
- except (FileNotFoundError, subprocess.CalledProcessError):
- if not os.path.exists(TOOL):
- sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"')
- raise # re-raise
-
- update_file_with_tmpfile(frozenfile, tmpfile, create=True)
-
-
 #######################################
 # the script
 
@@ -810,15 +720,7 @@ def main():
  # Regen build-related files.
  regen_makefile(modules)
  regen_pcbuild(modules)
-
- # Freeze the target modules.
- tmpsuffix = f'.{int(time.time())}'
- for src in _iter_sources(modules):
- _freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix)
-
- # Regen files dependent of frozen file details.
  regen_frozen(modules)
- regen_manifest(modules)
 
 
 if __name__ == '__main__':