[v3-0-test] Install python from source again (#53150)

.dockerignore

@@ -38,6 +38,7 @@
 !providers/
 !task-sdk/
 !airflow-ctl/
+!go-sdk/
 
 # Add all "test" distributions
 !tests

.github/workflows/additional-ci-image-checks.yml

@@ -150,4 +150,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin
       - name: "Check that image builds quickly"
-        run: breeze shell --max-time 600 --platform "${PLATFORM}"
+        run: breeze shell --max-time 900 --platform "${PLATFORM}"

.github/workflows/basic-tests.yml

@@ -293,6 +293,9 @@ jobs:
           --hook-stage manual update-installers-and-pre-commit || true
         if: always()
         env:
+          UPGRADE_UV: "true"
+          UPGRADE_PYTHON: "false"
+          UPGRADE_GOLANG: "true"
           UPGRADE_PIP: "false"
           UPGRADE_PRE_COMMIT: "false"
           UPGRADE_NODE_LTS: "false"
@@ -303,6 +306,9 @@ jobs:
           --hook-stage manual update-installers-and-pre-commit
         if: always()
         env:
+          UPGRADE_UV: "false"
+          UPGRADE_PYTHON: "true"
+          UPGRADE_GOLANG: "false"
           UPGRADE_PIP: "true"
           UPGRADE_PRE_COMMIT: "true"
           UPGRADE_NODE_LTS: "true"

.pre-commit-config.yaml

@@ -204,7 +204,7 @@ repos:
         files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$
         pass_filenames: false
         require_serial: true
-        additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0']
+        additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=25"]
       - id: update-chart-dependencies
         name: Update chart dependencies to latest (manual)
         entry: ./scripts/ci/pre_commit/update_chart_dependencies.py

Dockerfile.ci

@@ -16,13 +16,13 @@
 #
 # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
 #
-ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
+ARG BASE_IMAGE="debian:bookworm-slim"
 
 ##############################################################################################
 # This is the script image where we keep all inlined bash scripts needed in other segments
-# We use PYTHON_BASE_IMAGE to make sure that the scripts are different for different platforms.
+# We use BASE_IMAGE to make sure that the scripts are different for different platforms.
 ##############################################################################################
-FROM ${PYTHON_BASE_IMAGE} as scripts
+FROM ${BASE_IMAGE} as scripts
 
 ##############################################################################################
 # Please DO NOT modify the inlined scripts manually. The content of those files will be
@@ -31,22 +31,27 @@ FROM ${PYTHON_BASE_IMAGE} as scripts
 # make the PROD Dockerfile standalone
 ##############################################################################################
 
-# The content below is automatically copied from scripts/docker/install_os_dependencies.sh
-COPY <<"EOF" /install_os_dependencies.sh
+# The content below is automatically copied from scripts/docker/install_os_dependencies_ci.sh
+COPY <<"EOF" /install_os_dependencies_ci.sh
 #!/usr/bin/env bash
 set -euo pipefail
 
 if [[ "$#" != 1 ]]; then
-    echo "ERROR! There should be 'runtime' or 'dev' parameter passed as argument.".
+    echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.".
     exit 1
 fi
 
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10}
+GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
+
 if [[ "${1}" == "runtime" ]]; then
     INSTALLATION_TYPE="RUNTIME"
 elif   [[ "${1}" == "dev" ]]; then
-    INSTALLATION_TYPE="dev"
+    INSTALLATION_TYPE="DEV"
+elif   [[ "${1}" == "ci" ]]; then
+    INSTALLATION_TYPE="CI"
 else
-    echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime' or 'dev'.".
+    echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.".
     exit 1
 fi
 
@@ -56,7 +61,10 @@ function get_dev_apt_deps() {
 freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \
 libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \
 libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \
+gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
+libzstd-dev"
         export DEV_APT_DEPS
     fi
 }
@@ -143,14 +151,36 @@ function install_debian_runtime_dependencies() {
     rm -rf /var/lib/apt/lists/* /var/log/*
 }
 
+function install_python() {
+    git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1 https://github.com/python/cpython.git
+    cd cpython
+    ./configure --enable-optimizations
+    make -s -j "$(nproc)" all
+    make -s -j "$(nproc)" install
+    ln -s /usr/local/bin/python3 /usr/local/bin/python
+    ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+    cd ..
+    rm -rf cpython
+}
+
+function install_golang() {
+    curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
+    rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
+}
+
 if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
     get_runtime_apt_deps
     install_debian_runtime_dependencies
     install_docker_cli
 
 else
+
     get_dev_apt_deps
     install_debian_dev_dependencies
+    install_python
+    if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
+        install_golang
+    fi
     install_docker_cli
 fi
 EOF
@@ -936,7 +966,7 @@ function environment_initialization() {
     CI=${CI:="false"}
 
     # Added to have run-tests on path
-    export PATH=${PATH}:${AIRFLOW_SOURCES}
+    export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
 
     mkdir -pv "${AIRFLOW_HOME}/logs/"
 
@@ -1240,13 +1270,13 @@ COPY <<"EOF" /entrypoint_exec.sh
 exec /bin/bash "${@}"
 EOF
 
-FROM ${PYTHON_BASE_IMAGE} as main
+FROM ${BASE_IMAGE} as main
 
 # Nolog bash flag is currently ignored - but you can replace it with other flags (for example
 # xtrace - to show commands executed)
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 
-ARG PYTHON_BASE_IMAGE
+ARG BASE_IMAGE
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
 # By increasing this number we can do force build of all dependencies.
@@ -1256,7 +1286,7 @@ ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 ARG DEPENDENCIES_EPOCH_NUMBER="15"
 
 # Make sure noninteractive debian install is used and language variables set
-ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
+ENV BASE_IMAGE=${BASE_IMAGE} \
     DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
     LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
     DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
@@ -1267,7 +1297,7 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
     UV_CACHE_DIR=/root/.cache/uv
 
 
-RUN echo "Base image version: ${PYTHON_BASE_IMAGE}"
+RUN echo "Base image version: ${BASE_IMAGE}"
 
 ARG DEV_APT_COMMAND=""
 ARG ADDITIONAL_DEV_APT_COMMAND=""
@@ -1282,8 +1312,13 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
     ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
     ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
 
-COPY --from=scripts install_os_dependencies.sh /scripts/docker/
-RUN bash /scripts/docker/install_os_dependencies.sh dev
+ARG AIRFLOW_PYTHON_VERSION=v3.10.18
+ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION
+ENV GOLANG_MAJOR_MINOR_VERSION=1.24.4
+
+COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/
+
+RUN bash /scripts/docker/install_os_dependencies_ci.sh ci
 
 COPY --from=scripts common.sh /scripts/docker/
 

dev/breeze/src/airflow_breeze/global_constants.py

@@ -753,6 +753,16 @@ def generate_provider_dependencies_if_needed():
     },
 ]
 
+ALL_PYTHON_VERSION_TO_PATCH_VERSION: dict[str, str] = {
+    "3.6": "v3.6.1",
+    "3.7": "v3.7.1",
+    "3.8": "v3.8.1",
+    "3.9": "v3.9.23",
+    "3.10": "v3.10.18",
+    "3.11": "v3.11.13",
+    "3.12": "v3.12.11",
+}
+
 # Number of slices for low dep tests
 NUMBER_OF_LOW_DEP_SLICES = 5
 

dev/breeze/src/airflow_breeze/params/build_ci_params.py

@@ -21,6 +21,7 @@
 from pathlib import Path
 
 from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+from airflow_breeze.global_constants import ALL_PYTHON_VERSION_TO_PATCH_VERSION
 from airflow_breeze.params.common_build_params import CommonBuildParams
 from airflow_breeze.utils.path_utils import BUILD_CACHE_PATH
 
@@ -68,6 +69,9 @@ def prepare_arguments_for_docker_build_command(self) -> list[str]:
             self._opt_arg("UV_HTTP_TIMEOUT", get_uv_timeout(self))
         self._req_arg("AIRFLOW_VERSION", self.airflow_version)
         self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image)
+        self._req_arg(
+            "AIRFLOW_PYTHON_VERSION", ALL_PYTHON_VERSION_TO_PATCH_VERSION.get(self.python, self.python)
+        )
         if self.upgrade_to_newer_dependencies:
             self._opt_arg("UPGRADE_RANDOM_INDICATOR_STRING", f"{random.randrange(2**32):x}")
         # optional build args

scripts/ci/pre_commit/update_installers_and_pre_commit.py

@@ -24,6 +24,7 @@
 from pathlib import Path
 
 import requests
+from packaging.version import Version
 
 sys.path.insert(0, str(Path(__file__).parent.resolve()))  # make sure common_precommit_utils is imported
 from common_precommit_utils import AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH, console
@@ -65,6 +66,35 @@ def get_latest_pypi_version(package_name: str) -> str:
     return latest_version
 
 
+def get_latest_python_version(python_major_minor: str, github_token: str | None) -> str | None:
+    latest_version = None
+    # Matches versions of vA.B.C and vA.B where C can only be numeric and v is optional
+    version_match = re.compile(rf"^v?{python_major_minor}\.?\d*$")
+    headers = {"User-Agent": "Python requests"}
+    if github_token:
+        headers["Authorization"] = f"Bearer {github_token}"
+    for i in range(5):
+        response = requests.get(
+            f"https://api.github.com/repos/python/cpython/tags?per_page=100&page={i + 1}",
+            headers=headers,
+        )
+        response.raise_for_status()  # Ensure we got a successful response
+        data = response.json()
+        versions = [str(tag["name"]) for tag in data if version_match.match(tag.get("name", ""))]
+        if versions:
+            latest_version = sorted(versions, key=Version, reverse=True)[0]
+            break
+    return latest_version
+
+
+def get_latest_golang_version() -> str:
+    response = requests.get("https://go.dev/dl/?mode=json")
+    response.raise_for_status()  # Ensure we got a successful response
+    versions = response.json()
+    stable_versions = [release["version"].replace("go", "") for release in versions if release["stable"]]
+    return sorted(stable_versions, key=Version, reverse=True)[0]
+
+
 def get_latest_lts_node_version() -> str:
     response = requests.get("https://nodejs.org/dist/index.json")
     response.raise_for_status()  # Ensure we got a successful response
@@ -92,6 +122,15 @@ class Quoting(Enum):
     (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED),
 ]
 
+PYTHON_PATTERNS: list[tuple[str, Quoting]] = [
+    (r"(\"{python_major_minor}\": \")(v[0-9.]+)(\")", Quoting.UNQUOTED),
+]
+
+GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+    (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+    (re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED),
+]
+
 UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
     (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
     (re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED),
@@ -167,6 +206,8 @@ def get_replacement(value: str, quoting: Quoting) -> str:
 
 UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true"
 UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true"
+UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() == "true"
+UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() == "true"
 UPGRADE_SETUPTOOLS: bool = os.environ.get("UPGRADE_SETUPTOOLS", "true").lower() == "true"
 UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT", "true").lower() == "true"
 UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() == "true"
@@ -175,6 +216,10 @@ def get_replacement(value: str, quoting: Quoting) -> str:
 UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", "true").lower() == "true"
 UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", "true").lower() == "true"
 
+ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+
+GITHUB_TOKEN: str | None = os.environ.get("GITHUB_TOKEN")
+
 
 def replace_version(pattern: re.Pattern[str], version: str, text: str, keep_total_length: bool = True) -> str:
     # Assume that the pattern has up to 3 replacement groups:
@@ -205,6 +250,7 @@ def replacer(match):
 
 if __name__ == "__main__":
     changed = False
+    golang_version = get_latest_golang_version()
     pip_version = get_latest_pypi_version("pip")
     uv_version = get_latest_pypi_version("uv")
     setuptools_version = get_latest_pypi_version("setuptools")
@@ -225,6 +271,28 @@ def replacer(match):
                 new_content = replace_version(
                     line_pattern, get_replacement(pip_version, quoting), new_content, keep_length
                 )
+        if UPGRADE_PYTHON:
+            for python_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS:
+                latest_python_version = get_latest_python_version(python_version, GITHUB_TOKEN)
+                if latest_python_version:
+                    console.print(
+                        f"[bright_blue]Latest python {python_version} version: {latest_python_version}"
+                    )
+                    for line_format, quoting in PYTHON_PATTERNS:
+                        line_pattern = re.compile(line_format.format(python_major_minor=python_version))
+                        console.print(line_pattern)
+                        new_content = replace_version(
+                            line_pattern,
+                            get_replacement(latest_python_version, quoting),
+                            new_content,
+                            keep_length,
+                        )
+        if UPGRADE_GOLANG:
+            console.print(f"[bright_blue]Latest golang version: {golang_version}")
+            for line_pattern, quoting in GOLANG_PATTERNS:
+                new_content = replace_version(
+                    line_pattern, get_replacement(golang_version, quoting), new_content, keep_length
+                )
         if UPGRADE_SETUPTOOLS:
             console.print(f"[bright_blue]Latest setuptools version: {setuptools_version}")
             for line_pattern, quoting in SETUPTOOLS_PATTERNS:

scripts/docker/entrypoint_ci.sh

@@ -130,7 +130,7 @@ function environment_initialization() {
     CI=${CI:="false"}
 
     # Added to have run-tests on path
-    export PATH=${PATH}:${AIRFLOW_SOURCES}
+    export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
 
     mkdir -pv "${AIRFLOW_HOME}/logs/"