diff --git a/Dockerfile b/Dockerfile index bd6f5a56ccbfe..6f2e9c24d43a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -477,6 +477,26 @@ function install_airflow_dependencies_from_branch_tip() { set +x ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow set -x + # If you want to make sure dependency is removed from cache in your PR when you removed it from + # pyproject.toml - please add your dependency here as a list of strings + # for example: + # DEPENDENCIES_TO_REMOVE=("package_a" "package_b") + # Once your PR is merged, you should make a follow-up PR to remove it from this list + # and increase the AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci to make sure your cache is rebuilt. + local DEPENDENCIES_TO_REMOVE + # IMPORTANT!! Make sure to increase AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci when you remove a dependency from that list + DEPENDENCIES_TO_REMOVE=() + if [[ "${DEPENDENCIES_TO_REMOVE[*]}" != "" ]]; then + echo + echo "${COLOR_BLUE}Uninstalling just removed dependencies (temporary until cache refreshes)${COLOR_RESET}" + echo "${COLOR_BLUE}Dependencies to uninstall: ${DEPENDENCIES_TO_REMOVE[*]}${COLOR_RESET}" + echo + set +x + ${PACKAGING_TOOL_CMD} uninstall "${DEPENDENCIES_TO_REMOVE[@]}" || true + set -x + # make sure that the dependency is not needed by something else + pip check + fi } common::get_colors diff --git a/Dockerfile.ci b/Dockerfile.ci index 8c1227795bc9f..3e6ba2cc411e6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -424,6 +424,26 @@ function install_airflow_dependencies_from_branch_tip() { set +x ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow set -x + # If you want to make sure dependency is removed from cache in your PR when you removed it from + # pyproject.toml - please add your dependency here as a list of strings + # for example: + # DEPENDENCIES_TO_REMOVE=("package_a" "package_b") + # Once your PR is merged, you should make a follow-up PR to remove it from this list + # and increase the AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci to make sure your cache is rebuilt. + local DEPENDENCIES_TO_REMOVE + # IMPORTANT!! Make sure to increase AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci when you remove a dependency from that list + DEPENDENCIES_TO_REMOVE=() + if [[ "${DEPENDENCIES_TO_REMOVE[*]}" != "" ]]; then + echo + echo "${COLOR_BLUE}Uninstalling just removed dependencies (temporary until cache refreshes)${COLOR_RESET}" + echo "${COLOR_BLUE}Dependencies to uninstall: ${DEPENDENCIES_TO_REMOVE[*]}${COLOR_RESET}" + echo + set +x + ${PACKAGING_TOOL_CMD} uninstall "${DEPENDENCIES_TO_REMOVE[@]}" || true + set -x + # make sure that the dependency is not needed by something else + pip check + fi } common::get_colors diff --git a/pyproject.toml b/pyproject.toml index d15e9f2a7935c..8cd9cbf098518 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,10 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Topic :: System :: Monitoring", ] +# When you remove a dependency from the list, you should also make sure to add the dependency to be removed +# in the scripts/docker/install_airflow_dependencies_from_branch_tip.sh script DEPENDENCIES_TO_REMOVE +# in order to make sure the dependency is not installed in the CI image build process from the main +# of Airflow branch. After your PR is merged, you should remove it from the list there. dependencies = [ # Alembic is important to handle our migrations in predictable and performant way. It is developed # together with SQLAlchemy. Our experience with Alembic is that it very stable in minor version diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index f9c76f091a3e7..8158ab5886aa2 100644 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -67,6 +67,26 @@ function install_airflow_dependencies_from_branch_tip() { set +x ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow set -x + # If you want to make sure dependency is removed from cache in your PR when you removed it from + # pyproject.toml - please add your dependency here as a list of strings + # for example: + # DEPENDENCIES_TO_REMOVE=("package_a" "package_b") + # Once your PR is merged, you should make a follow-up PR to remove it from this list + # and increase the AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci to make sure your cache is rebuilt. + local DEPENDENCIES_TO_REMOVE + # IMPORTANT!! Make sure to increase AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci when you remove a dependency from that list + DEPENDENCIES_TO_REMOVE=() + if [[ "${DEPENDENCIES_TO_REMOVE[*]}" != "" ]]; then + echo + echo "${COLOR_BLUE}Uninstalling just removed dependencies (temporary until cache refreshes)${COLOR_RESET}" + echo "${COLOR_BLUE}Dependencies to uninstall: ${DEPENDENCIES_TO_REMOVE[*]}${COLOR_RESET}" + echo + set +x + ${PACKAGING_TOOL_CMD} uninstall "${DEPENDENCIES_TO_REMOVE[@]}" || true + set -x + # make sure that the dependency is not needed by something else + pip check + fi } common::get_colors