From 9a5b8b16ff6bea41d7bda1a7e851b13472aa837e Mon Sep 17 00:00:00 2001
From: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
Date: Sun, 25 May 2025 11:34:40 +0200
Subject: [PATCH] [v3-0-test] Bugfix/fix latest pypi version check (#51039)

* Fix version check from Pypi, requires user agent else raises HTTP 406

* Fix version check from Pypi, requires user agent else raises HTTP 406, also other cases in codebase
(cherry picked from commit eb7c346dd0dcd510fd99f2382aec81d41dc4d0c3)

Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
---
 dev/breeze/src/airflow_breeze/utils/version_utils.py         | 4 +++-
 dev/validate_version_added_fields_in_config.py               | 2 +-
 .../docker_tests/test_examples_of_prod_image_building.py     | 4 +++-
 scripts/ci/airflow_version_check.py                          | 5 +++--
 scripts/ci/pre_commit/update_installers_and_pre_commit.py    | 4 +++-
 5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/utils/version_utils.py b/dev/breeze/src/airflow_breeze/utils/version_utils.py
index 43650a108879c..eabee9a4b44ef 100644
--- a/dev/breeze/src/airflow_breeze/utils/version_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/version_utils.py
@@ -31,7 +31,9 @@ def get_latest_helm_chart_version():
 def get_latest_airflow_version():
     import requests
 
-    response = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    response = requests.get(
+        "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()
     latest_released_version = response.json()["info"]["version"]
     return latest_released_version
diff --git a/dev/validate_version_added_fields_in_config.py b/dev/validate_version_added_fields_in_config.py
index 7ee0fb16b2dcb..e4814d7a346d4 100755
--- a/dev/validate_version_added_fields_in_config.py
+++ b/dev/validate_version_added_fields_in_config.py
@@ -46,7 +46,7 @@
 
 
 def fetch_pypi_versions() -> list[str]:
-    r = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    r = requests.get("https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"})
     r.raise_for_status()
     all_version = r.json()["releases"].keys()
     released_versions = [d for d in all_version if not (("rc" in d) or ("b" in d))]
diff --git a/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py b/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
index c92c52e265772..888123db4eef2 100644
--- a/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
+++ b/docker-tests/tests/docker_tests/test_examples_of_prod_image_building.py
@@ -42,7 +42,9 @@
 
 @cache
 def get_latest_airflow_image():
-    response = requests.get("https://pypi.org/pypi/apache-airflow/json")
+    response = requests.get(
+        "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()
     latest_released_version = response.json()["info"]["version"]
     return f"apache/airflow:{latest_released_version}"
diff --git a/scripts/ci/airflow_version_check.py b/scripts/ci/airflow_version_check.py
index 6d2655fafcea1..8c040e13d024b 100755
--- a/scripts/ci/airflow_version_check.py
+++ b/scripts/ci/airflow_version_check.py
@@ -45,10 +45,11 @@ def check_airflow_version(airflow_version: Version) -> tuple[str, bool]:
     returns: tuple containing the version and a boolean indicating if it's latest.
     """
     latest = False
-    url = "https://pypi.org/pypi/apache-airflow/json"
     max_versions_shown = 30
     try:
-        response = requests.get(url)
+        response = requests.get(
+            "https://pypi.org/pypi/apache-airflow/json", headers={"User-Agent": "Python requests"}
+        )
         response.raise_for_status()
         data = response.json()
         latest_version = Version(data["info"]["version"])
diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
index 1f4b07f77e59b..ccc75e13ebc1e 100755
--- a/scripts/ci/pre_commit/update_installers_and_pre_commit.py
+++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
@@ -55,7 +55,9 @@
 
 
 def get_latest_pypi_version(package_name: str) -> str:
-    response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
+    response = requests.get(
+        f"https://pypi.org/pypi/{package_name}/json", headers={"User-Agent": "Python requests"}
+    )
     response.raise_for_status()  # Ensure we got a successful response
     data = response.json()
     latest_version = data["info"]["version"]  # The version info is under the 'info' key