From 15d3ad01ec4c1aa39d7ac6557900c4e5dd7daa24 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 11 Jul 2025 15:49:00 +0200 Subject: [PATCH] [v3-0-test] Move --no-binary flag to uv command line from pyproject.toml (#53179) When the flag is specified in pyproject toml, it also forces no binary installation of xmlsec and lxml for local uv sync which might fail if some system libraries are not installed, so it is better to do it in the image by passing the right flags to installer directly. (cherry picked from commit 47bbe55355757953507b8ae64cd05f9380e16748) Co-authored-by: Jarek Potiuk --- Dockerfile | 21 ++++++++++--- Dockerfile.ci | 31 ++++++++++++++----- scripts/docker/common.sh | 9 ++++-- scripts/docker/entrypoint_ci.sh | 10 ++++-- .../install_airflow_when_building_images.sh | 12 +++++-- 5 files changed, 64 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 023fb2442946c..141e6e129d7b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -473,11 +473,14 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then # for uv only install dev group when we install from sources - export EXTRA_INSTALL_FLAGS="--group=dev" + export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec" else - export EXTRA_INSTALL_FLAGS="" + export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec" fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" @@ -875,8 +878,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec else # We only use uv here but Installing using constraints is not supported with `uv sync`, so we # do not use ``uv sync`` because we are not committing and using uv.lock yet. @@ -933,8 +940,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-packag xmlsec set +x fi fi diff --git a/Dockerfile.ci b/Dockerfile.ci index 7fc0d572bf0c0..8168cc5187cf8 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -412,11 +412,14 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then # for uv only install dev group when we install from sources - export EXTRA_INSTALL_FLAGS="--group=dev" + export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec" else - export EXTRA_INSTALL_FLAGS="" + export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec" fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" @@ -628,8 +631,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec else # We only use uv here but Installing using constraints is not supported with `uv sync`, so we # do not use ``uv sync`` because we are not committing and using uv.lock yet. @@ -686,8 +693,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-packag xmlsec set +x fi fi @@ -1125,13 +1136,19 @@ function check_force_lowest_dependencies() { exit 0 fi cd "${AIRFLOW_SOURCES}/providers/${provider_id/.//}" || exit 1 - uv sync --resolution lowest-direct + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec else echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for Airflow.${COLOR_RESET}" echo cd "${AIRFLOW_SOURCES}/airflow-core" - uv sync --resolution lowest-direct + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec fi } diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index 4568aa2183b14..259ac7a76ec2e 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -42,11 +42,14 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then # for uv only install dev group when we install from sources - export EXTRA_INSTALL_FLAGS="--group=dev" + export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec" else - export EXTRA_INSTALL_FLAGS="" + export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec" fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index e62e0a58c8002..b8a4e994ceb21 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -336,13 +336,19 @@ function check_force_lowest_dependencies() { exit 0 fi cd "${AIRFLOW_SOURCES}/providers/${provider_id/.//}" || exit 1 - uv sync --resolution lowest-direct + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec else echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for Airflow.${COLOR_RESET}" echo cd "${AIRFLOW_SOURCES}/airflow-core" - uv sync --resolution lowest-direct + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 + uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec fi } diff --git a/scripts/docker/install_airflow_when_building_images.sh b/scripts/docker/install_airflow_when_building_images.sh index 3291a6b4cc0b4..78ddc7eaf8589 100644 --- a/scripts/docker/install_airflow_when_building_images.sh +++ b/scripts/docker/install_airflow_when_building_images.sh @@ -56,8 +56,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --resolution highest --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec else # We only use uv here but Installing using constraints is not supported with `uv sync`, so we # do not use ``uv sync`` because we are not committing and using uv.lock yet. @@ -114,8 +118,12 @@ function install_from_sources() { echo echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}" echo + # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 + # (binary lxml embeds its own libxml2, while xmlsec uses system one). + # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x - uv sync --all-packages --group dev --group docs --group docs-gen --group leveldb ${extra_sync_flags} + uv sync --all-packages --group dev --group docs --group docs-gen \ + --group leveldb ${extra_sync_flags} --no-binary-package lxml --no-binary-packag xmlsec set +x fi fi