diff --git a/.dockerignore b/.dockerignore index c50ed5ae24ee6..75e6291445ab6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -38,6 +38,7 @@ !providers/ !task-sdk/ !airflow-ctl/ +!go-sdk/ # Add all "test" distributions !tests diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 024f4a4ea0a93..5709e658bfd7e 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -150,4 +150,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin - name: "Check that image builds quickly" - run: breeze shell --max-time 600 --platform "${PLATFORM}" + run: breeze shell --max-time 900 --platform "${PLATFORM}" diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index bc23674f44285..24c0e8eb02e2a 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -293,6 +293,9 @@ jobs: --hook-stage manual update-installers-and-pre-commit || true if: always() env: + UPGRADE_UV: "true" + UPGRADE_PYTHON: "false" + UPGRADE_GOLANG: "true" UPGRADE_PIP: "false" UPGRADE_PRE_COMMIT: "false" UPGRADE_NODE_LTS: "false" @@ -303,6 +306,9 @@ jobs: --hook-stage manual update-installers-and-pre-commit if: always() env: + UPGRADE_UV: "false" + UPGRADE_PYTHON: "true" + UPGRADE_GOLANG: "false" UPGRADE_PIP: "true" UPGRADE_PRE_COMMIT: "true" UPGRADE_NODE_LTS: "true" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3adeefd79b661..bdafd8421e59d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -204,7 +204,7 @@ repos: files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$ pass_filenames: false require_serial: true - additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0'] + additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=25"] - id: update-chart-dependencies name: Update chart dependencies to latest (manual) entry: ./scripts/ci/pre_commit/update_chart_dependencies.py diff --git a/Dockerfile.ci b/Dockerfile.ci index 8168cc5187cf8..f30a72b1ec480 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -16,13 +16,13 @@ # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # -ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" +ARG BASE_IMAGE="debian:bookworm-slim" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments -# We use PYTHON_BASE_IMAGE to make sure that the scripts are different for different platforms. +# We use BASE_IMAGE to make sure that the scripts are different for different platforms. ############################################################################################## -FROM ${PYTHON_BASE_IMAGE} as scripts +FROM ${BASE_IMAGE} as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be @@ -31,22 +31,27 @@ FROM ${PYTHON_BASE_IMAGE} as scripts # make the PROD Dockerfile standalone ############################################################################################## -# The content below is automatically copied from scripts/docker/install_os_dependencies.sh -COPY <<"EOF" /install_os_dependencies.sh +# The content below is automatically copied from scripts/docker/install_os_dependencies_ci.sh +COPY <<"EOF" /install_os_dependencies_ci.sh #!/usr/bin/env bash set -euo pipefail if [[ "$#" != 1 ]]; then - echo "ERROR! There should be 'runtime' or 'dev' parameter passed as argument.". + echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.". exit 1 fi +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10} +GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} + if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" elif [[ "${1}" == "dev" ]]; then - INSTALLATION_TYPE="dev" + INSTALLATION_TYPE="DEV" +elif [[ "${1}" == "ci" ]]; then + INSTALLATION_TYPE="CI" else - echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime' or 'dev'.". + echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.". exit 1 fi @@ -56,7 +61,10 @@ function get_dev_apt_deps() { freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \ libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \ -software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev" +software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \ +gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ +libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \ +libzstd-dev" export DEV_APT_DEPS fi } @@ -143,14 +151,36 @@ function install_debian_runtime_dependencies() { rm -rf /var/lib/apt/lists/* /var/log/* } +function install_python() { + git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1 https://github.com/python/cpython.git + cd cpython + ./configure --enable-optimizations + make -s -j "$(nproc)" all + make -s -j "$(nproc)" install + ln -s /usr/local/bin/python3 /usr/local/bin/python + ln -s /usr/local/bin/pip3 /usr/local/bin/pip + cd .. + rm -rf cpython +} + +function install_golang() { + curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" + rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz +} + if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then get_runtime_apt_deps install_debian_runtime_dependencies install_docker_cli else + get_dev_apt_deps install_debian_dev_dependencies + install_python + if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then + install_golang + fi install_docker_cli fi EOF @@ -936,7 +966,7 @@ function environment_initialization() { CI=${CI:="false"} # Added to have run-tests on path - export PATH=${PATH}:${AIRFLOW_SOURCES} + export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/ mkdir -pv "${AIRFLOW_HOME}/logs/" @@ -1240,13 +1270,13 @@ COPY <<"EOF" /entrypoint_exec.sh exec /bin/bash "${@}" EOF -FROM ${PYTHON_BASE_IMAGE} as main +FROM ${BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example # xtrace - to show commands executed) SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] -ARG PYTHON_BASE_IMAGE +ARG BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies. @@ -1256,7 +1286,7 @@ ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG DEPENDENCIES_EPOCH_NUMBER="15" # Make sure noninteractive debian install is used and language variables set -ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ +ENV BASE_IMAGE=${BASE_IMAGE} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \ DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \ @@ -1267,7 +1297,7 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ UV_CACHE_DIR=/root/.cache/uv -RUN echo "Base image version: ${PYTHON_BASE_IMAGE}" +RUN echo "Base image version: ${BASE_IMAGE}" ARG DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_APT_COMMAND="" @@ -1282,8 +1312,13 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} -COPY --from=scripts install_os_dependencies.sh /scripts/docker/ -RUN bash /scripts/docker/install_os_dependencies.sh dev +ARG AIRFLOW_PYTHON_VERSION=v3.10.18 +ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION +ENV GOLANG_MAJOR_MINOR_VERSION=1.24.4 + +COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/ + +RUN bash /scripts/docker/install_os_dependencies_ci.sh ci COPY --from=scripts common.sh /scripts/docker/ diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 7e0f5fff5f9dc..c680ddca5a8c0 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -753,6 +753,16 @@ def generate_provider_dependencies_if_needed(): }, ] +ALL_PYTHON_VERSION_TO_PATCH_VERSION: dict[str, str] = { + "3.6": "v3.6.1", + "3.7": "v3.7.1", + "3.8": "v3.8.1", + "3.9": "v3.9.23", + "3.10": "v3.10.18", + "3.11": "v3.11.13", + "3.12": "v3.12.11", +} + # Number of slices for low dep tests NUMBER_OF_LOW_DEP_SLICES = 5 diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index 65adcf6efde9d..b4c42c341ded0 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -21,6 +21,7 @@ from pathlib import Path from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.global_constants import ALL_PYTHON_VERSION_TO_PATCH_VERSION from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.utils.path_utils import BUILD_CACHE_PATH @@ -68,6 +69,9 @@ def prepare_arguments_for_docker_build_command(self) -> list[str]: self._opt_arg("UV_HTTP_TIMEOUT", get_uv_timeout(self)) self._req_arg("AIRFLOW_VERSION", self.airflow_version) self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image) + self._req_arg( + "AIRFLOW_PYTHON_VERSION", ALL_PYTHON_VERSION_TO_PATCH_VERSION.get(self.python, self.python) + ) if self.upgrade_to_newer_dependencies: self._opt_arg("UPGRADE_RANDOM_INDICATOR_STRING", f"{random.randrange(2**32):x}") # optional build args diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py b/scripts/ci/pre_commit/update_installers_and_pre_commit.py index 612acdaaa2f2f..eed555e93fe40 100755 --- a/scripts/ci/pre_commit/update_installers_and_pre_commit.py +++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py @@ -24,6 +24,7 @@ from pathlib import Path import requests +from packaging.version import Version sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported from common_precommit_utils import AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH, console @@ -65,6 +66,35 @@ def get_latest_pypi_version(package_name: str) -> str: return latest_version +def get_latest_python_version(python_major_minor: str, github_token: str | None) -> str | None: + latest_version = None + # Matches versions of vA.B.C and vA.B where C can only be numeric and v is optional + version_match = re.compile(rf"^v?{python_major_minor}\.?\d*$") + headers = {"User-Agent": "Python requests"} + if github_token: + headers["Authorization"] = f"Bearer {github_token}" + for i in range(5): + response = requests.get( + f"https://api.github.com/repos/python/cpython/tags?per_page=100&page={i + 1}", + headers=headers, + ) + response.raise_for_status() # Ensure we got a successful response + data = response.json() + versions = [str(tag["name"]) for tag in data if version_match.match(tag.get("name", ""))] + if versions: + latest_version = sorted(versions, key=Version, reverse=True)[0] + break + return latest_version + + +def get_latest_golang_version() -> str: + response = requests.get("https://go.dev/dl/?mode=json") + response.raise_for_status() # Ensure we got a successful response + versions = response.json() + stable_versions = [release["version"].replace("go", "") for release in versions if release["stable"]] + return sorted(stable_versions, key=Version, reverse=True)[0] + + def get_latest_lts_node_version() -> str: response = requests.get("https://nodejs.org/dist/index.json") response.raise_for_status() # Ensure we got a successful response @@ -92,6 +122,15 @@ class Quoting(Enum): (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), ] +PYTHON_PATTERNS: list[tuple[str, Quoting]] = [ + (r"(\"{python_major_minor}\": \")(v[0-9.]+)(\")", Quoting.UNQUOTED), +] + +GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), +] + UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), (re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED), @@ -167,6 +206,8 @@ def get_replacement(value: str, quoting: Quoting) -> str: UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" +UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() == "true" +UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() == "true" UPGRADE_SETUPTOOLS: bool = os.environ.get("UPGRADE_SETUPTOOLS", "true").lower() == "true" UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT", "true").lower() == "true" UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() == "true" @@ -175,6 +216,10 @@ def get_replacement(value: str, quoting: Quoting) -> str: UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", "true").lower() == "true" UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", "true").lower() == "true" +ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + +GITHUB_TOKEN: str | None = os.environ.get("GITHUB_TOKEN") + def replace_version(pattern: re.Pattern[str], version: str, text: str, keep_total_length: bool = True) -> str: # Assume that the pattern has up to 3 replacement groups: @@ -205,6 +250,7 @@ def replacer(match): if __name__ == "__main__": changed = False + golang_version = get_latest_golang_version() pip_version = get_latest_pypi_version("pip") uv_version = get_latest_pypi_version("uv") setuptools_version = get_latest_pypi_version("setuptools") @@ -225,6 +271,28 @@ def replacer(match): new_content = replace_version( line_pattern, get_replacement(pip_version, quoting), new_content, keep_length ) + if UPGRADE_PYTHON: + for python_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS: + latest_python_version = get_latest_python_version(python_version, GITHUB_TOKEN) + if latest_python_version: + console.print( + f"[bright_blue]Latest python {python_version} version: {latest_python_version}" + ) + for line_format, quoting in PYTHON_PATTERNS: + line_pattern = re.compile(line_format.format(python_major_minor=python_version)) + console.print(line_pattern) + new_content = replace_version( + line_pattern, + get_replacement(latest_python_version, quoting), + new_content, + keep_length, + ) + if UPGRADE_GOLANG: + console.print(f"[bright_blue]Latest golang version: {golang_version}") + for line_pattern, quoting in GOLANG_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(golang_version, quoting), new_content, keep_length + ) if UPGRADE_SETUPTOOLS: console.print(f"[bright_blue]Latest setuptools version: {setuptools_version}") for line_pattern, quoting in SETUPTOOLS_PATTERNS: diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index b8a4e994ceb21..831be7e241fca 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -130,7 +130,7 @@ function environment_initialization() { CI=${CI:="false"} # Added to have run-tests on path - export PATH=${PATH}:${AIRFLOW_SOURCES} + export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/ mkdir -pv "${AIRFLOW_HOME}/logs/" diff --git a/scripts/docker/install_os_dependencies_ci.sh b/scripts/docker/install_os_dependencies_ci.sh new file mode 100644 index 0000000000000..0873ac69c8c4f --- /dev/null +++ b/scripts/docker/install_os_dependencies_ci.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck shell=bash +set -euo pipefail + +if [[ "$#" != 1 ]]; then + echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.". + exit 1 +fi + +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10} +GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} + +if [[ "${1}" == "runtime" ]]; then + INSTALLATION_TYPE="RUNTIME" +elif [[ "${1}" == "dev" ]]; then + INSTALLATION_TYPE="DEV" +elif [[ "${1}" == "ci" ]]; then + INSTALLATION_TYPE="CI" +else + echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.". + exit 1 +fi + +function get_dev_apt_deps() { + if [[ "${DEV_APT_DEPS=}" == "" ]]; then + DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \ +freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4 libev-dev libffi-dev libgeos-dev \ +libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ +libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client pkgconf sasl2-bin \ +software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \ +gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ +libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \ +libzstd-dev" + export DEV_APT_DEPS + fi +} + +function get_runtime_apt_deps() { + local debian_version + local debian_version_apt_deps + # Get debian version without installing lsb_release + # shellcheck disable=SC1091 + debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) + echo + echo "DEBIAN CODENAME: ${debian_version}" + echo + debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd" + echo + echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" + echo + if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then + RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \ +curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \ +ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales ${debian_version_apt_deps} \ +lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo unixodbc" + export RUNTIME_APT_DEPS + fi +} + +function install_docker_cli() { + apt-get update + apt-get install ca-certificates curl + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + # shellcheck disable=SC1091 + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-get install -y --no-install-recommends docker-ce-cli +} + +function install_debian_dev_dependencies() { + apt-get update + apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 + apt-get install -y --no-install-recommends curl gnupg2 lsb-release + # shellcheck disable=SC2086 + export ${ADDITIONAL_DEV_APT_ENV?} + if [[ ${DEV_APT_COMMAND} != "" ]]; then + bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" + fi + if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then + bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" + fi + apt-get update + local debian_version + local debian_version_apt_deps + # Get debian version without installing lsb_release + # shellcheck disable=SC1091 + debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) + echo + echo "DEBIAN CODENAME: ${debian_version}" + echo + # shellcheck disable=SC2086 + apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS} +} + +function install_debian_runtime_dependencies() { + apt-get update + apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 + apt-get install -y --no-install-recommends curl gnupg2 lsb-release + # shellcheck disable=SC2086 + export ${ADDITIONAL_RUNTIME_APT_ENV?} + if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then + bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" + fi + if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then + bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" + fi + apt-get update + # shellcheck disable=SC2086 + apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} + apt-get autoremove -yqq --purge + apt-get clean + rm -rf /var/lib/apt/lists/* /var/log/* +} + +function install_python() { + git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1 https://github.com/python/cpython.git + cd cpython + ./configure --enable-optimizations + make -s -j "$(nproc)" all + make -s -j "$(nproc)" install + ln -s /usr/local/bin/python3 /usr/local/bin/python + ln -s /usr/local/bin/pip3 /usr/local/bin/pip + cd .. + rm -rf cpython +} + +function install_golang() { + curl "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" + rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz +} + +if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then + get_runtime_apt_deps + install_debian_runtime_dependencies + install_docker_cli + +else + + get_dev_apt_deps + install_debian_dev_dependencies + install_python + if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then + install_golang + fi + install_docker_cli +fi