Passing arch to the nix install script and adjusting brew libs + use macos-14 to build if needed

.github/workflows/build-python-packages.yml

@@ -15,7 +15,7 @@ on:
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,macos-11_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -39,7 +39,7 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,macos-11,macos-14_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {

builders/build-python.ps1

@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform
 $Builder.Build()

builders/macos-python-builder.psm1

@@ -31,10 +31,10 @@ class macOSPythonBuilder : NixPythonBuilder {
         .SYNOPSIS
         Prepare system environment by installing dependencies and required packages.
         #>
-        
+
         if ($this.Version -eq "3.7.17") {
-            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
-            # setting up an environemnt 
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for
+            # setting up an environment
             # If we get any issues realted to ncurses or readline we can try to run this command
             # brew install ncurses readline
             Execute-Command -Command "brew install bzip2"
@@ -68,21 +68,37 @@ class macOSPythonBuilder : NixPythonBuilder {
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
-            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
+            $env:LDFLAGS = "-L$(brew --prefix openssl@1.1)/lib -L$(brew --prefix zlib)/lib"
+            $env:CFLAGS = "-I$(brew --prefix openssl@1.1)/include -I$(brew --prefix zlib)/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+            $configureString += " --with-openssl=$(brew --prefix openssl@1.1)"
+
+            # Configure may detect libintl from non-system sources, such as Homebrew (it **does** on macos arm64) so turn it off
+            # $configureString += " ac_cv_lib_intl_textdomain=no"
+            # This has libintl.a in there, so hopefully it picks it up
+            $env:LDFLAGS = "-L$(brew --prefix gettext)/lib"
+            $env:CFLAGS = "-I$(brew --prefix gettext)/include"
 
             # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
             # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
             if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
-                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
-                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+                $env:LDFLAGS += " -L$(brew --prefix zlib)/lib"
+                $env:CFLAGS += " -I$(brew --prefix zlib)/include"
             }
 
-            if ($this.Version -gt "3.7.12") {
-                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
-	        }
+            if ($this.Version -ge "3.11.0") {
+                # Python 3.11+: configure: WARNING: unrecognized options: --with-tcltk-includes, --with-tcltk-libs
+                # https://github.com/python/cpython/blob/762f489b31afe0f0589aa784bf99c752044e7f30/Doc/whatsnew/3.11.rst#L2167-L2172
+                $tcl_inc_dir = "$(brew --prefix tcl-tk)/include"
+                # In Homebrew Tcl/Tk 8.6.13, headers have been moved to the 'tcl-tk' subdir
+                if (Test-Path -Path "$tcl_inc_dir/tcl-tk") {
+                    $tcl_inc_dir = "$tcl_inc_dir/tcl-tk"
+                }
+                $env:TCLTK_CFLAGS = "-I$tcl_inc_dir"
+                $env:TCLTK_LIBS = "-L$(brew --prefix tcl-tk)/lib -ltcl8.6 -ltk8.6"
+            } elseif ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I $(brew --prefix tcl-tk)/include' --with-tcltk-libs='-L$(brew --prefix tcl-tk)/lib -ltcl8.6 -ltk8.6'"
+            }
 
             if ($this.Version -eq "3.7.17") {
                 $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
@@ -101,6 +117,12 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         Write-Host "The passed configure options are: "
         Write-Host $configureString
+        Write-Host "Flags: "
+        Write-Host "CFLAGS='$env:CFLAGS'"
+        Write-Host "CPPFLAGS='$env:CPPFLAGS'"
+        Write-Host "LDFLAGS='$env:LDFLAGS'"
+        Write-Host "TCLTK_CFLAGS='$env:TCLTK_CFLAGS'"
+        Write-Host "TCLTK_LIBS='$env:TCLTK_LIBS'"
 
         Execute-Command -Command $configureString
     }
@@ -180,7 +202,7 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         $PkgVersion = [semver]"3.11.0-beta.1"
 
-        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+        if (($this.Version -ge $PkgVersion) -and ($this.Architecture -eq "x64"))  {
             Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
             $this.DownloadPkg()
 

builders/nix-python-builder.psm1

@@ -20,11 +20,11 @@ class NixPythonBuilder : PythonBuilder {
     .PARAMETER InstallationTemplateName
     The name of template that will be used to create installation script for generated Python artifact.
 
-    .PARAMETER InstallationScriptName	
+    .PARAMETER InstallationScriptName
     The name of installation script that will be generated for Python artifact.
 
     .PARAMETER OutputArtifactName
-    The name of archive with Python binaries that will be generated as part of Python artifact. 
+    The name of archive with Python binaries that will be generated as part of Python artifact.
 
     #>
 
@@ -37,7 +37,7 @@ class NixPythonBuilder : PythonBuilder {
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
-        $this.InstallationTemplateName = "nix-setup-template.sh"	
+        $this.InstallationTemplateName = "nix-setup-template.sh"
         $this.InstallationScriptName = "setup.sh"
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
@@ -97,6 +97,7 @@ class NixPythonBuilder : PythonBuilder {
 
         $variablesToReplace = @{
             "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__ARCH__}}" = $this.Architecture;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 
@@ -133,7 +134,7 @@ class NixPythonBuilder : PythonBuilder {
     [void] Build() {
         <#
         .SYNOPSIS
-        Build Python artifact from sources. 
+        Build Python artifact from sources.
         #>
 
         Write-Host "Prepare Python Hostedtoolcache location..."

installers/nix-setup-template.sh

@@ -1,5 +1,6 @@
 set -e
 
+ARCH="{{__ARCH__}}"
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@@ -17,7 +18,7 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 
 echo "Check if Python hostedtoolcache folder exist..."
 if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
@@ -54,4 +55,4 @@ export PIP_ROOT_USER_ACTION=ignore
 ./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/$ARCH.complete

tests/sources/python-config-test.py

@@ -28,7 +28,7 @@
 if pkg_installer:
     expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
 else:
-    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'
 
 # Check modules
 ### Validate libraries path