diff --git a/.dockerignore b/.dockerignore index 75e6291445ab6..c50ed5ae24ee6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -38,7 +38,6 @@ !providers/ !task-sdk/ !airflow-ctl/ -!go-sdk/ # Add all "test" distributions !tests diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 5c395c3c30268..161ba36d58043 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -292,8 +292,6 @@ jobs: if: always() env: UPGRADE_UV: "true" - UPGRADE_PYTHON: "false" - UPGRADE_GOLANG: "true" UPGRADE_PIP: "false" UPGRADE_PRE_COMMIT: "false" UPGRADE_NODE_LTS: "false" @@ -305,8 +303,6 @@ jobs: if: always() env: UPGRADE_UV: "false" - UPGRADE_PYTHON: "true" - UPGRADE_GOLANG: "false" UPGRADE_PIP: "true" UPGRADE_PRE_COMMIT: "true" UPGRADE_NODE_LTS: "true" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5af7b636d3754..13b7121241e9d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -213,7 +213,7 @@ repos: ^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$ pass_filenames: false require_serial: true - additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=25"] + additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0'] - id: update-chart-dependencies name: Update chart dependencies to latest (manual) entry: ./scripts/ci/pre_commit/update_chart_dependencies.py diff --git a/Dockerfile.ci b/Dockerfile.ci index 9e7c1afd6a2de..6ba138121af1a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -16,13 +16,13 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -ARG BASE_IMAGE="debian:bookworm-slim" +ARG PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments -# We use BASE_IMAGE to make sure that the scripts are different for different platforms. +# We use PYTHON_BASE_IMAGE to make sure that the scripts are different for different platforms. ############################################################################################## -FROM ${BASE_IMAGE} as scripts +FROM ${PYTHON_BASE_IMAGE} as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be @@ -31,27 +31,22 @@ FROM ${BASE_IMAGE} as scripts # make the PROD Dockerfile standalone ############################################################################################## -# The content below is automatically copied from scripts/docker/install_os_dependencies_ci.sh -COPY <<"EOF" /install_os_dependencies_ci.sh +# The content below is automatically copied from scripts/docker/install_os_dependencies.sh +COPY <<"EOF" /install_os_dependencies.sh #!/usr/bin/env bash set -euo pipefail if [[ "$#" != 1 ]]; then - echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.". + echo "ERROR! There should be 'runtime' or 'dev' parameter passed as argument.". exit 1 fi -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10} -GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} - if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" elif [[ "${1}" == "dev" ]]; then - INSTALLATION_TYPE="DEV" -elif [[ "${1}" == "ci" ]]; then - INSTALLATION_TYPE="CI" + INSTALLATION_TYPE="dev" else - echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.". + echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime' or 'dev'.". exit 1 fi @@ -61,10 +56,7 @@ function get_dev_apt_deps() { freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \ libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \ -software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \ -gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ -libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \ -libzstd-dev" +software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev" export DEV_APT_DEPS fi } @@ -151,35 +143,14 @@ function install_debian_runtime_dependencies() { rm -rf /var/lib/apt/lists/* /var/log/* } -function install_python() { - git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1 https://github.com/python/cpython.git - cd cpython - ./configure --enable-optimizations - make -s -j "$(nproc)" install - ln -s /usr/local/bin/python3 /usr/local/bin/python - ln -s /usr/local/bin/pip3 /usr/local/bin/pip - cd .. - rm -rf cpython -} - -function install_golang() { - curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" - rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz -} - if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then get_runtime_apt_deps install_debian_runtime_dependencies install_docker_cli else - get_dev_apt_deps install_debian_dev_dependencies - install_python - if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then - install_golang - fi install_docker_cli fi EOF @@ -954,7 +925,7 @@ function environment_initialization() { CI=${CI:="false"} # Added to have run-tests on path - export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/ + export PATH=${PATH}:${AIRFLOW_SOURCES} mkdir -pv "${AIRFLOW_HOME}/logs/" @@ -1266,13 +1237,13 @@ COPY <<"EOF" /entrypoint_exec.sh exec /bin/bash "${@}" EOF -FROM ${BASE_IMAGE} as main +FROM ${PYTHON_BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example # xtrace - to show commands executed) SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] -ARG BASE_IMAGE +ARG PYTHON_BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies. @@ -1282,7 +1253,7 @@ ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG DEPENDENCIES_EPOCH_NUMBER="15" # Make sure noninteractive debian install is used and language variables set -ENV BASE_IMAGE=${BASE_IMAGE} \ +ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \ DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \ @@ -1293,7 +1264,7 @@ ENV BASE_IMAGE=${BASE_IMAGE} \ UV_CACHE_DIR=/root/.cache/uv -RUN echo "Base image version: ${BASE_IMAGE}" +RUN echo "Base image version: ${PYTHON_BASE_IMAGE}" ARG DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_APT_COMMAND="" @@ -1308,12 +1279,8 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} -ENV AIRFLOW_PYTHON_VERSION=v3.10.18 -ENV GOLANG_MAJOR_MINOR_VERSION=1.24.4 - -COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/ - -RUN bash /scripts/docker/install_os_dependencies_ci.sh ci +COPY --from=scripts install_os_dependencies.sh /scripts/docker/ +RUN bash /scripts/docker/install_os_dependencies.sh dev COPY --from=scripts common.sh /scripts/docker/ diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py b/scripts/ci/pre_commit/update_installers_and_pre_commit.py index 8613949f7b306..ef1b745f9fe5d 100755 --- a/scripts/ci/pre_commit/update_installers_and_pre_commit.py +++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py @@ -24,7 +24,6 @@ from pathlib import Path import requests -from packaging.version import Version sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported from common_precommit_utils import AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH, console @@ -66,35 +65,6 @@ def get_latest_pypi_version(package_name: str) -> str: return latest_version -def get_latest_python_version(python_major_minor: str, github_token: str | None) -> str | None: - latest_version = None - # Matches versions of vA.B.C and vA.B where C can only be numeric and v is optional - version_match = re.compile(rf"^v?{python_major_minor}\.?\d*$") - headers = {"User-Agent": "Python requests"} - if github_token: - headers["Authorization"] = f"Bearer {github_token}" - for i in range(5): - response = requests.get( - f"https://api.github.com/repos/python/cpython/tags?per_page=100&page={i + 1}", - headers=headers, - ) - response.raise_for_status() # Ensure we got a successful response - data = response.json() - versions = [str(tag["name"]) for tag in data if version_match.match(tag.get("name", ""))] - if versions: - latest_version = sorted(versions, key=Version, reverse=True)[0] - break - return latest_version - - -def get_latest_golang_version() -> str: - response = requests.get("https://go.dev/dl/?mode=json") - response.raise_for_status() # Ensure we got a successful response - versions = response.json() - stable_versions = [release["version"].replace("go", "") for release in versions if release["stable"]] - return sorted(stable_versions, key=Version, reverse=True)[0] - - def get_latest_lts_node_version() -> str: response = requests.get("https://nodejs.org/dist/index.json") response.raise_for_status() # Ensure we got a successful response @@ -122,16 +92,6 @@ class Quoting(Enum): (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), ] -PYTHON_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(AIRFLOW_PYTHON_VERSION=)(v[0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(\| *`AIRFLOW_PYTHON_VERSION` *\| *)(`v[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), -] - -GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), -] - UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), (re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED), @@ -207,16 +167,10 @@ def get_replacement(value: str, quoting: Quoting) -> str: UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" -UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() == "true" -UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() == "true" UPGRADE_SETUPTOOLS: bool = os.environ.get("UPGRADE_SETUPTOOLS", "true").lower() == "true" UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT", "true").lower() == "true" UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() == "true" -PYTHON_VERSION: str = os.environ.get("PYTHON_VERSION", "3.10") - -GITHUB_TOKEN: str | None = os.environ.get("GITHUB_TOKEN") - def replace_version(pattern: re.Pattern[str], version: str, text: str, keep_total_length: bool = True) -> str: # Assume that the pattern has up to 3 replacement groups: @@ -247,8 +201,6 @@ def replacer(match): if __name__ == "__main__": changed = False - python_version = get_latest_python_version(PYTHON_VERSION, GITHUB_TOKEN) - golang_version = get_latest_golang_version() pip_version = get_latest_pypi_version("pip") uv_version = get_latest_pypi_version("uv") setuptools_version = get_latest_pypi_version("setuptools") @@ -265,18 +217,6 @@ def replacer(match): new_content = replace_version( line_pattern, get_replacement(pip_version, quoting), new_content, keep_length ) - if UPGRADE_PYTHON and python_version: - console.print(f"[bright_blue]Latest python {PYTHON_VERSION} version: {python_version}") - for line_pattern, quoting in PYTHON_PATTERNS: - new_content = replace_version( - line_pattern, get_replacement(python_version, quoting), new_content, keep_length - ) - if UPGRADE_GOLANG: - console.print(f"[bright_blue]Latest golang version: {golang_version}") - for line_pattern, quoting in GOLANG_PATTERNS: - new_content = replace_version( - line_pattern, get_replacement(golang_version, quoting), new_content, keep_length - ) if UPGRADE_SETUPTOOLS: console.print(f"[bright_blue]Latest setuptools version: {setuptools_version}") for line_pattern, quoting in SETUPTOOLS_PATTERNS: diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index fef63aa88f068..e215ce642a0df 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -130,7 +130,7 @@ function environment_initialization() { CI=${CI:="false"} # Added to have run-tests on path - export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/ + export PATH=${PATH}:${AIRFLOW_SOURCES} mkdir -pv "${AIRFLOW_HOME}/logs/" diff --git a/scripts/docker/install_os_dependencies_ci.sh b/scripts/docker/install_os_dependencies_ci.sh deleted file mode 100644 index b2e6294dbe07c..0000000000000 --- a/scripts/docker/install_os_dependencies_ci.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck shell=bash -set -euo pipefail - -if [[ "$#" != 1 ]]; then - echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.". - exit 1 -fi - -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10} -GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} - -if [[ "${1}" == "runtime" ]]; then - INSTALLATION_TYPE="RUNTIME" -elif [[ "${1}" == "dev" ]]; then - INSTALLATION_TYPE="DEV" -elif [[ "${1}" == "ci" ]]; then - INSTALLATION_TYPE="CI" -else - echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.". - exit 1 -fi - -function get_dev_apt_deps() { - if [[ "${DEV_APT_DEPS=}" == "" ]]; then - DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \ -freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \ -libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ -libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \ -software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \ -gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ -libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \ -libzstd-dev" - export DEV_APT_DEPS - fi -} - -function get_runtime_apt_deps() { - local debian_version - local debian_version_apt_deps - # Get debian version without installing lsb_release - # shellcheck disable=SC1091 - debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) - echo - echo "DEBIAN CODENAME: ${debian_version}" - echo - debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" - echo - echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" - echo - if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then - RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \ -curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \ -ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales ${debian_version_apt_deps} \ -lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo unixodbc" - export RUNTIME_APT_DEPS - fi -} - -function install_docker_cli() { - apt-get update - apt-get install ca-certificates curl - install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc - chmod a+r /etc/apt/keyrings/docker.asc - # shellcheck disable=SC1091 - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - apt-get update - apt-get install -y --no-install-recommends docker-ce-cli -} - -function install_debian_dev_dependencies() { - apt-get update - apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends curl gnupg2 lsb-release - # shellcheck disable=SC2086 - export ${ADDITIONAL_DEV_APT_ENV?} - if [[ ${DEV_APT_COMMAND} != "" ]]; then - bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" - fi - if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then - bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" - fi - apt-get update - local debian_version - local debian_version_apt_deps - # Get debian version without installing lsb_release - # shellcheck disable=SC1091 - debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) - echo - echo "DEBIAN CODENAME: ${debian_version}" - echo - # shellcheck disable=SC2086 - apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS} -} - -function install_debian_runtime_dependencies() { - apt-get update - apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 - apt-get install -y --no-install-recommends curl gnupg2 lsb-release - # shellcheck disable=SC2086 - export ${ADDITIONAL_RUNTIME_APT_ENV?} - if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then - bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" - fi - if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then - bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" - fi - apt-get update - # shellcheck disable=SC2086 - apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} - apt-get autoremove -yqq --purge - apt-get clean - rm -rf /var/lib/apt/lists/* /var/log/* -} - -function install_python() { - git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1 https://github.com/python/cpython.git - cd cpython - ./configure --enable-optimizations - make -s -j "$(nproc)" install - ln -s /usr/local/bin/python3 /usr/local/bin/python - ln -s /usr/local/bin/pip3 /usr/local/bin/pip - cd .. - rm -rf cpython -} - -function install_golang() { - curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" - rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz -} - -if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then - get_runtime_apt_deps - install_debian_runtime_dependencies - install_docker_cli - -else - - get_dev_apt_deps - install_debian_dev_dependencies - install_python - if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then - install_golang - fi - install_docker_cli -fi