Skip to content

Add more dependency reports #52606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Jul 1, 2025
Merged

Add more dependency reports #52606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions .github/workflows/finalize-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,11 @@ jobs:
runs-on: ${{ fromJSON(inputs.runners) }}
needs: [update-constraints]
if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64'
name: "Dependency upgrade summary"
name: "Dependencies ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
strategy:
matrix:
python-version: ${{ fromJson(inputs.python-versions) }}
constraints-mode: ["constraints", "constraints-source-providers", "constraints-no-providers"]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
Expand All @@ -151,14 +155,18 @@ jobs:
uses: ./.github/actions/prepare_breeze_and_image
with:
platform: ${{ inputs.platform }}
python: ${{ inputs.default-python-version }}
python: ${{ matrix.python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Dependency upgrade summary"
shell: bash
run: >
breeze run uv run /opt/airflow/dev/constraints-updated-version-check.py --explain-why
breeze run uv run /opt/airflow/dev/constraints-updated-version-check.py
--python-version "${MATRIX_PYTHON_VERSION}"
--airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" --explain-why
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
MATRIX_PYTHON_VERSION: "${{ matrix.python-version }}"
MATRIX_CONSTRAINTS_MODE: "${{ matrix.constraints-mode }}"
VERBOSE: "false"

push-buildx-cache-to-github-registry:
name: Push Regular Image Cache ${{ inputs.platform }}
Expand Down
85 changes: 52 additions & 33 deletions dev/constraints-updated-version-check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,20 @@ def parse_constraints_generation_date(lines):
return None


def get_constraints_file(python_version):
url = f"https://raw.githubusercontent.com/apache/airflow/refs/heads/constraints-main/constraints-{python_version}.txt"
def get_constraints_file(python_version, airflow_constraints_mode="constraints"):
url = f"https://raw.githubusercontent.com/apache/airflow/refs/heads/constraints-main/{airflow_constraints_mode}-{python_version}.txt"
try:
response = urllib.request.urlopen(url)
return response.read().decode("utf-8").splitlines()
except HTTPError as e:
if e.code == 404:
console.print(f"[bold red]Error: Constraints file for Python {python_version} not found.[/]")
console.print(
f"[bold red]Error: Constraints file for Python {python_version} and mode {airflow_constraints_mode} not found.[/]"
)
console.print(f"[bold red]URL: {url}[/]")
console.print("[bold red]Please check if the Python version is correct and the file exists.[/]")
console.print(
"[bold red]Please check if the Python version and constraints mode are correct and the file exists.[/]"
)
else:
console.print(f"[bold red]HTTP Error: {e.code} - {e.reason}[/]")
exit(1)
Expand Down Expand Up @@ -163,13 +167,16 @@ def get_first_newer_release_date_str(releases, current_version):

def main(
python_version: str,
airflow_constraints_mode: str,
mode: str,
selected_packages: set[str] | None = None,
explain_why: bool = False,
verbose: bool = False,
):
lines = get_constraints_file(python_version)
lines = get_constraints_file(python_version, airflow_constraints_mode)
constraints_date = parse_constraints_generation_date(lines)
console.print(f"[bold cyan]Python version:[/] [white]{python_version}[/]")
console.print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]\n")
if constraints_date:
console.print(
f"[bold cyan]Constraints file generation date:[/] [white]{constraints_date.strftime('%Y-%m-%d %H:%M:%S')}[/]"
Expand All @@ -181,7 +188,7 @@ def main(
print_table_header(format_str, headers, total_width)

outdated_count, skipped_count, explanations = process_packages(
packages, constraints_date, mode, explain_why, verbose, col_widths, format_str
packages, constraints_date, mode, explain_why, verbose, col_widths, format_str, python_version
)

print_table_footer(total_width, len(packages), outdated_count, skipped_count, mode)
Expand Down Expand Up @@ -267,6 +274,7 @@ def process_packages(
verbose: bool,
col_widths: dict,
format_str: str,
python_version: str,
) -> tuple[int, int, list[str]]:
import subprocess
import tempfile
Expand Down Expand Up @@ -319,6 +327,10 @@ def update_pyproject_dependency(pyproject_path: Path, pkg: str, latest_version:
if not dep_added:
new_lines.append(f' "{pkg}=={latest_version}",')
pyproject_path.write_text("\n".join(new_lines) + "\n")
if verbose:
console.print(
f"[cyan]Fixed {pkg} at {latest_version} in [white]{pyproject_path}[/] [dim](pyproject.toml)[/]"
)

airflow_pyproject = Path(__file__).parent.parent / "pyproject.toml"
airflow_pyproject = airflow_pyproject.resolve()
Expand Down Expand Up @@ -369,6 +381,7 @@ def update_pyproject_dependency(pyproject_path: Path, pkg: str, latest_version:
run_uv_sync,
update_pyproject_dependency,
verbose,
python_version,
)
explanations.append(explanation)
except HTTPError as e:
Expand Down Expand Up @@ -430,6 +443,7 @@ def explain_package_upgrade(
run_uv_sync,
update_pyproject_dependency,
verbose: bool,
python_version: str,
) -> str:
explanation = (
f"[bold blue]\n--- Explaining for {pkg} (current: {pinned_version}, latest: {latest_version}) ---[/]"
Expand All @@ -452,6 +466,8 @@ def preserve_pyproject_file(pyproject_path: Path):
"--resolution",
"highest",
"--refresh",
"--python",
python_version,
],
cwd=repo_root,
)
Expand All @@ -470,13 +486,15 @@ def preserve_pyproject_file(pyproject_path: Path):
"--resolution",
"highest",
"--refresh",
"--python",
python_version,
],
cwd=repo_root,
)
(temp_dir_path / "uv_sync_after.txt").write_text(after_result.stdout + after_result.stderr)
if after_result.returncode == 0:
explanation += f"\n[bold yellow]Package {pkg} can be upgraded from {pinned_version} to {latest_version} without conflicts.[/]."
else:
if after_result.returncode != 0 or verbose:
explanation += f"\n[yellow]uv sync output for {pkg}=={latest_version}:[/]\n"
explanation += after_result.stdout + after_result.stderr
return explanation
Expand All @@ -485,46 +503,47 @@ def preserve_pyproject_file(pyproject_path: Path):
@click.command()
@click.option(
"--python-version",
required=False,
type=click.Choice(["3.10", "3.11", "3.12", "3.13"]),
default="3.10",
help="Python version to check constraints for (e.g., 3.12)",
help="Python version to check constraints for (e.g. 3.10, 3.11, 3.12, 3.13).",
)
@click.option(
"--airflow-constraints-mode",
type=click.Choice(
["constraints", "constraints-no-providers", "constraints-source-providers"], case_sensitive=False
),
default="constraints",
show_default=True,
help="Constraints mode to use: constraints, constraints-no-providers, constraints-source-providers.",
)
@click.option(
"--mode",
type=click.Choice(["full", "diff-constraints", "diff-all"]),
default="diff-constraints",
type=click.Choice(["full", "diff-all", "diff-constraints"], case_sensitive=False),
default="full",
show_default=True,
help="Operation mode: full, diff-constraints, or diff-all.",
help="Report mode: full, diff-all, diff-constraints.",
)
@click.option(
"--selected-packages",
required=False,
default=None,
help="Comma-separated list of package names to check (e.g., 'requests,flask'). If not set, all packages are checked.",
"--package",
multiple=True,
help="Only check specific package(s). Can be used multiple times.",
)
@click.option(
"--explain-why",
is_flag=True,
"--explain-why/--no-explain-why",
default=False,
help="For each selected package, attempts to upgrade to the latest version and explains why it cannot be upgraded.",
help="Show explanations for outdated packages.",
)
@click.option(
"--verbose",
is_flag=True,
"--verbose/--no-verbose",
default=False,
help="Print the temporary pyproject.toml file used for each package when running --explain-why.",
help="Show verbose output.",
)
def cli(
python_version: str,
mode: str,
selected_packages: str | None,
explain_why: bool,
verbose: bool,
):
selected_packages_set = None
if selected_packages:
selected_packages_set = set(pkg.strip() for pkg in selected_packages.split(",") if pkg.strip())
main(python_version, mode, selected_packages_set, explain_why, verbose)
def cli(python_version, airflow_constraints_mode, mode, package, explain_why, verbose):
if os.environ.get("CI", "false") == "true":
# Show output outside the group in CI
print("::endgroup::")
selected_packages = set(package) if package else None
main(python_version, airflow_constraints_mode, mode, selected_packages, explain_why, verbose)


if __name__ == "__main__":
Expand Down