Limit requests in botocore upgrade test (#39747)

The botocore upgrade test brings also requests to latest version
even if it is limited in `[docker]` extra after #39740. This PR
adds similar limit when botocore is upgraded.

(cherry picked from commit 39269d6)

Dockerfile.ci

@@ -1005,8 +1005,12 @@ function check_boto_upgrade() {
     ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
     # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass
     # the pip check test, Similarly gcloud-aio-auth limit is needed to be included as it bumps cryptography
+    # Also until docker-py compatibility with requests 2.32 is fixed we need to limit requests version
+    # Should be removed after  https://github.com/docker/docker-py/issues/3256 together with removal of similar
+    # limitation in providers/docker/pyproject.toml
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0"
+    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \
+       "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0" "requests<2.32.0"
     pip check
 }
 

airflow/providers/docker/provider.yaml

@@ -69,7 +69,8 @@ dependencies:
   - python-dotenv>=0.21.0
   # requests 2.32.0 has an incompatibility with python lib docker
   # https://github.com/psf/requests/issues/6707 and https://github.com/docker/docker-py/issues/3256
-  # once resolved remove this dependency
+  # once resolved remove this dependency and bump docker-py version
+  # !!!!! Make sure to also remove it from the scripts/docker/entrypoint_ci.sh !!!!!
   - requests>=2.27.0,<2.32.0
 
 integrations:

scripts/docker/entrypoint_ci.sh

@@ -225,8 +225,12 @@ function check_boto_upgrade() {
     ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
     # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass
     # the pip check test, Similarly gcloud-aio-auth limit is needed to be included as it bumps cryptography
+    # Also until docker-py compatibility with requests 2.32 is fixed we need to limit requests version
+    # Should be removed after  https://github.com/docker/docker-py/issues/3256 together with removal of similar
+    # limitation in providers/docker/pyproject.toml
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0"
+    ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \
+       "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0" "requests<2.32.0"
     pip check
 }