bpo-45573: Introduce extension module flags in Makefile (GH-29594)

``configure`` now uses a standardized format to forward state, compiler
flags, and linker flags to ``Makefile``, ``setup.py``, and
``Modules/Setup``. ``makesetup`` use the new variables by default if a
module line does not contain any compiler or linker flags. ``setup.py``
has a new function ``addext()``.

For a module ``egg``, configure adds:

* ``MODULE_EGG`` with value yes, missing, disabled, or n/a
* ``MODULE_EGG_CFLAGS``
* ``MODULE_EGG_LDFLAGS``

``Makefile.pre.in`` may also provide ``MODULE_EGG_DEPS`` that lists
dependencies such as header files and static libs.

Signed-off-by: Christian Heimes <christian@python.org>

Author: tiran

Makefile.pre.in

@@ -201,12 +201,26 @@ RUNSHARED=       @RUNSHARED@
 # ensurepip options
 ENSUREPIP=      @ENSUREPIP@
 
+# Internal static libraries
+LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
+LIBEXPAT_A= Modules/expat/libexpat.a
+
 # OpenSSL options for setup.py so sysconfig can pick up AC_SUBST() vars.
 OPENSSL_INCLUDES=@OPENSSL_INCLUDES@
 OPENSSL_LIBS=@OPENSSL_LIBS@
 OPENSSL_LDFLAGS=@OPENSSL_LDFLAGS@
 OPENSSL_RPATH=@OPENSSL_RPATH@
 
+# Module compiler and linker flags
+#   yes: module is available
+#   missing: build dependency is missing
+#   disabled: module is disabled
+#   n/a: module is not available on the current platform
+# MODULE_EGG=yes  # yes, missing, disabled, n/a
+# MODULE_EGG_CFLAGS=
+# MODULE_EGG_LDFLAGS=
+@MODULE_BLOCK@
+
 # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var
 TZPATH=@TZPATH@
 
@@ -535,8 +549,6 @@ LIBMPDEC_HEADERS= \
 		$(srcdir)/Modules/_decimal/libmpdec/typearith.h \
 		$(srcdir)/Modules/_decimal/libmpdec/umodarith.h
 
-LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
-
 ##########################################################################
 # pyexpat's expat library
 
@@ -562,8 +574,6 @@ LIBEXPAT_HEADERS= \
 		Modules/expat/xmltok.h \
 		Modules/expat/xmltok_impl.h
 
-LIBEXPAT_A= Modules/expat/libexpat.a
-
 #########################################################################
 # Rules
 

Misc/NEWS.d/next/Build/2021-11-17-19-02-51.bpo-45573.GMNdun.rst

@@ -0,0 +1,3 @@
+``configure`` now uses a unified format to set state, compiler flags, and
+linker flags in Makefile. The new macro ``PY_STDLIB_MOD`` sets three
+variables that are consumed by ``Modules/Setup`` and ``setup.py``.

Modules/Setup

@@ -42,6 +42,15 @@
 # You can also use any Make variable that is detected by configure and
 # defined in Makefile.pre.in, e.g. OpenSSL flags $(OPENSSL_INCLUDES).
 #
+# Rules generated by makesetup use additional variables:
+#
+# - All source file rules have a dependency on $(PYTHON_HEADERS) and on
+#   optional variable $(MODULES_{mod_upper}_DEPS).
+# - If no <cpparg> and no <library> arguments are given, then makesetup
+#   defaults to $(MODULES_{mod_upper}_CFLAGS) cppargs and
+#   $(MODULES_{mod_upper}_LDFLAGS) libraries. The variables are typically
+#   defined by configure.
+#
 # The build process works like this:
 #
 # 1. Build all modules that are declared as static in Modules/Setup,
@@ -149,7 +158,7 @@ time timemodule.c
 #_contextvars _contextvarsmodule.c
 #_csv _csv.c
 #_datetime _datetimemodule.c
-#_decimal _decimal/_decimal.c $(DECIMAL_CFLAGS) $(DECIMAL_LDFLAGS)
+#_decimal _decimal/_decimal.c
 #_heapq _heapqmodule.c
 #_json _json.c
 #_lsprof _lsprof.c rotatingtree.c
@@ -172,8 +181,8 @@ time timemodule.c
 #select selectmodule.c
 
 # XML
-#_elementtree _elementtree.c $(EXPAT_CFLAGS)
-#pyexpat pyexpat.c $(EXPAT_CFLAGS) $(EXPAT_LDFLAGS)
+#_elementtree _elementtree.c
+#pyexpat pyexpat.c
 
 # hashing builtins
 #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c

Modules/makesetup

@@ -154,6 +154,7 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 		cpps=
 		libs=
 		mods=
+		mods_upper=
 		skip=
 		for arg in $line
 		do
@@ -194,11 +195,17 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 			*.*)		echo 1>&2 "bad word $arg in $line"
 					exit 1;;
 			-u)		skip=libs; libs="$libs -u";;
-			[a-zA-Z_]*)	mods="$mods $arg";;
+			[a-zA-Z_]*)
+					mods="$mods $arg"
+					mods_upper=$(echo $mods | tr '[a-z]' '[A-Z]');;
 			*)		echo 1>&2 "bad word $arg in $line"
 					exit 1;;
 			esac
 		done
+		if test -z "$cpps" -a -z "$libs"; then
+			cpps="\$(MODULE_${mods_upper}_CFLAGS)"
+			libs="\$(MODULE_${mods_upper}_LDFLAGS)"
+		fi
 		case $doconfig in
 		yes)
 			LIBS="$LIBS $libs"
@@ -245,7 +252,6 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 			*)
 				cc="$cc \$(PY_BUILTIN_MODULE_CFLAGS)";;
 			esac
-			mods_upper=$(echo $mods | tr '[a-z]' '[A-Z]')
 			# force rebuild when header file or module build flavor (static/shared) is changed
 			rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS) Modules/config.c; $cc $cpps -c $src -o $obj"
 			echo "$rule" >>$rulesf

aclocal.m4

@@ -619,3 +619,53 @@ AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
         [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
 ])dnl PKG_HAVE_DEFINE_WITH_MODULES
 
+# AM_CONDITIONAL                                            -*- Autoconf -*-
+
+# Copyright (C) 1997-2020 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_CONDITIONAL(NAME, SHELL-CONDITION)
+# -------------------------------------
+# Define a conditional.
+AC_DEFUN([AM_CONDITIONAL],
+[AC_PREREQ([2.52])dnl
+ m4_if([$1], [TRUE],  [AC_FATAL([$0: invalid condition: $1])],
+       [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl
+AC_SUBST([$1_TRUE])dnl
+AC_SUBST([$1_FALSE])dnl
+_AM_SUBST_NOTMAKE([$1_TRUE])dnl
+_AM_SUBST_NOTMAKE([$1_FALSE])dnl
+m4_define([_AM_COND_VALUE_$1], [$2])dnl
+if $2; then
+  $1_TRUE=
+  $1_FALSE='#'
+else
+  $1_TRUE='#'
+  $1_FALSE=
+fi
+AC_CONFIG_COMMANDS_PRE(
+[if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then
+  AC_MSG_ERROR([[conditional "$1" was never defined.
+Usually this means the macro was only invoked conditionally.]])
+fi])])
+
+# Copyright (C) 2006-2020 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# _AM_SUBST_NOTMAKE(VARIABLE)
+# ---------------------------
+# Prevent Automake from outputting VARIABLE = @VARIABLE@ in Makefile.in.
+# This macro is traced by Automake.
+AC_DEFUN([_AM_SUBST_NOTMAKE])
+
+# AM_SUBST_NOTMAKE(VARIABLE)
+# --------------------------
+# Public sister of _AM_SUBST_NOTMAKE.
+AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)])
+