From 8ed5d550597141dbdaa1b85c9d6ca670554fa8af Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 8 Aug 2025 06:18:55 +0200 Subject: [PATCH] Switch pre-commit to prek The pre-commit is a fantastic tool, and we heavily used it for years, but generally the tool stagnated and is not showing a sign of adapting to our needs. For years we tried to convince pre-commit maintainers that things like autocomplete are necessary - but it met with pretty much resistance (if not hostility) from the maintainer. Also there was no chance for them to accept expectations of bigger projects like ours, where we have a huge monorepo and not only multiple needs but also different parts of the repo needing different language support (golang, typescript soon) - and apparenty the maintainer of pre-commit does not think monorepo is a good thing at all. Similarly - they did not recognize the raise of `uv` and the only way to use `uv` with pre-commit is to patch it by installing `pre-comit-uv` that essentialy patches pre-commit with uv support. This is not really sustainable and the tool lags behind many of our needs. Luckily - we have new project in town - prek - which rewrites pre-commit that is 100% compatible (now), 10x faster (because rust), uses `uv` natively, supports auto-complete already and they have very friendly maintainer who is not only supporting us but also very happily works on improving `prek` to close all the gaps, and plans to implement (with our support of course and cooperation) monorepo support - that will allow us to modularise our pre-commits. This PR switches our pre-commit support to use prek exclusively: * breeze static checks command is completely removed * custom auto-complete code in breeze as well * instructions are updated to setup prek instead of precommit * CI is updated to run prek instead of pre-commmit * documentation for static checks is reviewed and new features that prek enables are added --- .github/.pre-commit-config.yaml | 2 +- .../action.yml | 57 +- .github/boring-cyborg.yml | 4 +- .../workflows/additional-ci-image-checks.yml | 4 +- .github/workflows/basic-tests.yml | 74 +- .github/workflows/ci-amd.yml | 28 +- .github/workflows/ci-arm.yml | 6 +- .github/workflows/ci-image-checks.yml | 46 +- .github/workflows/news-fragment.yml | 2 +- .gitpod.yml | 6 +- .pre-commit-config.yaml | 258 ++-- .rat-excludes | 3 + AGENTS.md | 6 +- Dockerfile | 17 +- Dockerfile.ci | 24 +- INSTALL | 18 +- PROVIDERS.rst | 4 +- README.md | 2 +- RELEASE_NOTES.rst | 2 +- airflow-core/docs/database-erd-ref.rst | 2 +- airflow-core/docs/howto/custom-operator.rst | 2 +- airflow-core/docs/howto/usage-cli.rst | 2 +- .../docs/installation/supported-versions.rst | 2 +- airflow-core/docs/migrations-ref.rst | 2 +- airflow-core/hatch_build.py | 2 +- airflow-core/pyproject.toml | 2 +- .../src/airflow/ui/public/i18n/README.md | 2 +- .../docs/installation/supported-versions.rst | 2 +- contributing-docs/02_how_to_communicate.rst | 2 +- .../03_contributors_quick_start.rst | 49 +- contributing-docs/05_pull_requests.rst | 14 +- .../06_development_environments.rst | 4 +- contributing-docs/07_local_virtualenv.rst | 4 +- contributing-docs/08_static_code_checks.rst | 610 ++------- .../12_provider_distributions.rst | 10 +- .../14_metadata_database_updates.rst | 10 +- contributing-docs/16_adding_api_endpoints.rst | 18 +- .../17_architecture_diagrams.rst | 12 +- .../18_contribution_workflow.rst | 6 +- .../testing/integration_tests.rst | 3 - dev/README_RELEASE_AIRFLOW.md | 4 +- dev/README_RELEASE_HELM_CHART.md | 2 +- dev/breeze/README.md | 4 +- dev/breeze/doc/01_installation.rst | 2 +- dev/breeze/doc/03_developer_tasks.rst | 45 +- dev/breeze/doc/04_troubleshooting.rst | 30 +- .../doc/07_breeze_maintenance_tasks.rst | 2 +- dev/breeze/doc/10_advanced_breeze_topics.rst | 15 +- dev/breeze/doc/ci/02_images.md | 5 +- dev/breeze/doc/ci/04_selective_checks.md | 18 +- dev/breeze/doc/ci/05_workflows.md | 2 +- dev/breeze/doc/ci/07_running_ci_locally.md | 6 +- .../doc/images/output-commands-hash.txt | 2 +- dev/breeze/doc/images/output-commands.svg | 66 +- ...agement_prepare-provider-documentation.svg | 28 +- ...agement_prepare-provider-documentation.txt | 2 +- dev/breeze/doc/images/output_run.svg | 4 +- dev/breeze/doc/images/output_run.txt | 2 +- ...utput_setup_check-all-params-in-groups.svg | 2 +- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 2 +- ...output_setup_regenerate-command-images.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 6 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 2 +- .../doc/images/output_start-airflow.txt | 2 +- .../doc/images/output_static-checks.svg | 432 ------- .../doc/images/output_static-checks.txt | 1 - dev/breeze/pyproject.toml | 3 +- .../commands/ci_image_commands.py | 4 +- .../airflow_breeze/commands/common_options.py | 2 +- .../commands/developer_commands.py | 196 +-- .../commands/developer_commands_config.py | 31 - .../airflow_breeze/commands/main_command.py | 51 +- .../commands/release_management_commands.py | 16 +- .../src/airflow_breeze/global_constants.py | 6 +- .../src/airflow_breeze/params/shell_params.py | 9 +- .../src/airflow_breeze/pre_commit_ids.py | 171 --- .../pre_commit_ids_TEMPLATE.py.jinja2 | 25 - .../provider_documentation.py | 2 +- .../templates/pyproject_TEMPLATE.toml.jinja2 | 2 +- .../utils/provider_dependencies.py | 2 +- .../src/airflow_breeze/utils/reinstall.py | 2 +- .../src/airflow_breeze/utils/run_utils.py | 62 +- .../airflow_breeze/utils/selective_checks.py | 28 +- dev/breeze/tests/test_selective_checks.py | 102 +- dev/breeze/uv.lock | 1090 +++++++++-------- devel-common/src/docs/README.rst | 4 +- .../src/tests_common/pytest_plugin.py | 2 +- .../tests/docker_tests/docker_utils.py | 2 +- docs/spelling_wordlist.txt | 1 + generated/PYPI_README.md | 2 +- providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 27 +- providers/airbyte/pyproject.toml | 2 +- providers/alibaba/pyproject.toml | 2 +- providers/amazon/pyproject.toml | 2 +- providers/apache/beam/pyproject.toml | 2 +- providers/apache/cassandra/pyproject.toml | 2 +- providers/apache/drill/pyproject.toml | 2 +- providers/apache/druid/pyproject.toml | 2 +- providers/apache/flink/pyproject.toml | 2 +- providers/apache/hdfs/pyproject.toml | 2 +- providers/apache/hive/pyproject.toml | 2 +- providers/apache/iceberg/pyproject.toml | 2 +- providers/apache/impala/pyproject.toml | 2 +- providers/apache/kafka/pyproject.toml | 2 +- providers/apache/kylin/pyproject.toml | 2 +- providers/apache/livy/pyproject.toml | 2 +- providers/apache/pig/pyproject.toml | 2 +- providers/apache/pinot/pyproject.toml | 2 +- providers/apache/spark/pyproject.toml | 2 +- .../system/apache/spark/example_pyspark.py | 2 +- providers/apache/tinkerpop/pyproject.toml | 2 +- providers/apprise/pyproject.toml | 2 +- providers/arangodb/pyproject.toml | 2 +- providers/asana/pyproject.toml | 2 +- providers/atlassian/jira/pyproject.toml | 2 +- providers/celery/pyproject.toml | 2 +- providers/cloudant/pyproject.toml | 2 +- providers/cncf/kubernetes/pyproject.toml | 2 +- providers/cohere/pyproject.toml | 2 +- providers/common/compat/pyproject.toml | 2 +- providers/common/io/pyproject.toml | 2 +- providers/common/messaging/pyproject.toml | 2 +- providers/common/sql/pyproject.toml | 2 +- .../providers/common/sql/README_API.md | 18 +- .../providers/common/sql/dialects/dialect.pyi | 2 +- .../common/sql/get_provider_info.pyi | 2 +- .../providers/common/sql/hooks/handlers.pyi | 2 +- .../providers/common/sql/hooks/sql.pyi | 2 +- .../common/sql/operators/generic_transfer.pyi | 2 +- .../providers/common/sql/sensors/sql.pyi | 2 +- .../providers/common/sql/triggers/sql.pyi | 2 +- providers/databricks/pyproject.toml | 2 +- providers/datadog/pyproject.toml | 2 +- providers/dbt/cloud/pyproject.toml | 2 +- providers/dingding/pyproject.toml | 2 +- providers/discord/pyproject.toml | 2 +- providers/docker/pyproject.toml | 2 +- providers/edge3/pyproject.toml | 2 +- providers/elasticsearch/pyproject.toml | 2 +- providers/exasol/pyproject.toml | 2 +- providers/fab/docs/migrations-ref.rst | 2 +- providers/fab/pyproject.toml | 2 +- providers/facebook/pyproject.toml | 2 +- providers/ftp/pyproject.toml | 2 +- providers/git/pyproject.toml | 2 +- providers/github/pyproject.toml | 2 +- providers/google/pyproject.toml | 2 +- providers/grpc/pyproject.toml | 2 +- providers/hashicorp/pyproject.toml | 2 +- providers/http/pyproject.toml | 2 +- providers/imap/pyproject.toml | 2 +- providers/influxdb/pyproject.toml | 2 +- providers/jdbc/pyproject.toml | 2 +- providers/jenkins/pyproject.toml | 2 +- providers/keycloak/pyproject.toml | 2 +- providers/microsoft/azure/pyproject.toml | 2 +- providers/microsoft/mssql/pyproject.toml | 2 +- providers/microsoft/psrp/pyproject.toml | 2 +- providers/microsoft/winrm/pyproject.toml | 2 +- providers/mongo/pyproject.toml | 2 +- providers/mysql/pyproject.toml | 2 +- providers/neo4j/pyproject.toml | 2 +- providers/odbc/pyproject.toml | 2 +- providers/openai/pyproject.toml | 2 +- providers/openfaas/pyproject.toml | 2 +- providers/openlineage/pyproject.toml | 2 +- providers/opensearch/pyproject.toml | 2 +- providers/opsgenie/pyproject.toml | 2 +- providers/oracle/pyproject.toml | 2 +- providers/pagerduty/pyproject.toml | 2 +- providers/papermill/pyproject.toml | 2 +- providers/pgvector/pyproject.toml | 2 +- providers/pinecone/pyproject.toml | 2 +- providers/postgres/pyproject.toml | 2 +- providers/presto/pyproject.toml | 2 +- providers/qdrant/pyproject.toml | 2 +- providers/redis/pyproject.toml | 2 +- providers/salesforce/pyproject.toml | 2 +- providers/samba/pyproject.toml | 2 +- providers/segment/pyproject.toml | 2 +- providers/sendgrid/pyproject.toml | 2 +- providers/sftp/pyproject.toml | 2 +- .../src/airflow/providers/sftp/hooks/sftp.py | 2 +- providers/singularity/pyproject.toml | 2 +- providers/slack/pyproject.toml | 2 +- providers/smtp/pyproject.toml | 2 +- providers/snowflake/pyproject.toml | 2 +- providers/sqlite/pyproject.toml | 2 +- providers/ssh/pyproject.toml | 2 +- providers/standard/pyproject.toml | 2 +- providers/tableau/pyproject.toml | 2 +- providers/telegram/pyproject.toml | 2 +- providers/teradata/pyproject.toml | 2 +- providers/trino/pyproject.toml | 2 +- providers/vertica/pyproject.toml | 2 +- providers/weaviate/pyproject.toml | 2 +- providers/yandex/pyproject.toml | 2 +- providers/ydb/pyproject.toml | 2 +- providers/zendesk/pyproject.toml | 2 +- pyproject.toml | 6 +- reproducible_build.yaml | 4 +- scripts/ci/docker-compose/remove-sources.yml | 6 +- scripts/ci/docker-compose/tests-sources.yml | 4 +- scripts/ci/install_breeze.sh | 2 +- .../ci/pre_commit/check_pre_commit_hooks.py | 163 --- scripts/ci/{pre_commit => prek}/.gitignore | 0 .../ci/{pre_commit => prek}/boring_cyborg.py | 0 .../{pre_commit => prek}/breeze_cmd_line.py | 4 +- .../capture_airflowctl_help.py | 6 +- .../changelog_duplicates.py | 0 .../ci/{pre_commit => prek}/chart_schema.py | 0 .../check_aiobotocore_optional.py | 0 .../check_airflow_bug_report_template.py | 4 +- .../check_airflow_imports.py | 4 +- .../check_airflow_v_imports_in_tests.py | 4 +- .../check_base_operator_partial_arguments.py | 6 +- .../check_changelog_format.py | 0 .../check_common_sql_dependency.py | 2 +- .../check_default_configuration.py | 6 +- .../check_deferrable_default.py | 0 .../check_deprecations.py | 0 .../check_extra_packages_ref.py | 8 +- .../{pre_commit => prek}/check_i18n_json.py | 8 +- .../check_imports_in_providers.py | 6 +- .../check_init_decorator_arguments.py | 6 +- .../check_init_in_tests.py | 0 .../check_integrations_list.py | 4 +- .../{pre_commit => prek}/check_kubeconform.py | 4 +- .../check_lazy_logging.py | 0 .../ci/{pre_commit => prek}/check_license.py | 2 +- .../check_min_python_version.py | 10 +- .../check_order_dockerfile_extras.py | 4 +- .../check_provider_docs.py | 2 +- .../check_provider_version_compat.py | 4 +- .../check_provider_yaml_files.py | 6 +- ...eck_providers_subpackages_all_have_init.py | 20 +- .../check_revision_heads_map.py | 6 +- .../{pre_commit => prek}/check_sdk_imports.py | 2 +- .../check_shared_distributions_structure.py | 4 +- .../check_shared_distributions_usage.py | 6 +- .../check_system_tests.py | 0 .../check_system_tests_hidden_in_index.py | 4 +- ...check_template_context_variable_in_sync.py | 6 +- .../check_template_fields.py | 6 +- .../check_tests_in_right_folders.py | 6 +- .../check_ti_vs_tis_attributes.py | 4 +- .../checkout_no_credentials.py | 0 .../common_prek_black_utils.py} | 4 +- .../common_prek_utils.py} | 12 +- .../compile_provider_assets.py | 8 +- .../{pre_commit => prek}/compile_ui_assets.py | 10 +- .../compile_ui_assets_dev.py | 10 +- ...corator_operator_implements_custom_name.py | 0 .../ci/{pre_commit => prek}/diagram_hash.txt | 0 .../docstring_param_type.py | 0 .../{pre_commit => prek}/draft7_schema.json | 0 .../generate_airflow_diagrams.py | 4 +- .../generate_openapi_spec.py | 6 +- .../generate_openapi_spec_fab.py | 6 +- .../generate_openapi_spec_keycloak.py | 6 +- .../generate_pypi_readme.py | 4 +- .../generate_volumes_for_sources.py | 8 +- .../inline_scripts_in_docker.py | 2 +- .../{pre_commit => prek}/lint_dockerfile.py | 2 +- scripts/ci/{pre_commit => prek}/lint_helm.py | 4 +- .../{pre_commit => prek}/lint_json_schema.py | 4 +- .../{pre_commit => prek}/local_yml_mounts.py | 4 +- .../migration_reference.py | 6 +- scripts/ci/{pre_commit => prek}/mypy.py | 6 +- .../ci/{pre_commit => prek}/mypy_folder.py | 8 +- .../new_session_in_provide_session.py | 0 .../ci/{pre_commit => prek}/newsfragments.py | 0 .../prevent_deprecated_sqlalchemy_usage.py | 0 .../replace_bad_characters.py | 0 .../ci/{pre_commit => prek}/ruff_format.py | 0 .../significant_newsfragments_checker.py | 0 .../{pre_commit => prek}/sort_in_the_wild.py | 2 +- .../sort_installed_providers.py | 2 +- .../sort_spelling_wordlist.py | 2 +- .../supported_versions.py | 0 .../ts_compile_lint_simple_auth_manager_ui.py | 4 +- .../ts_compile_lint_ui.py | 4 +- .../{pre_commit => prek}/unittest_testcase.py | 0 .../update_airflow_pyproject_toml.py | 4 +- .../update_black_version.py | 6 +- .../update_breeze_config_hash.py | 4 +- .../update_chart_dependencies.py | 4 +- .../{pre_commit => prek}/update_er_diagram.py | 8 +- .../update_example_dags_paths.py | 4 +- .../update_providers_build_files.py | 4 +- .../update_providers_dependencies.py | 4 +- .../update_source_date_epoch.py | 4 +- .../{pre_commit => prek}/update_versions.py | 4 +- .../upgrade_important_versions.py} | 70 +- .../validate_chart_annotations.py | 2 +- .../validate_operators_init.py | 0 .../vendor_k8s_json_schema.py | 0 scripts/docker/common.sh | 13 +- .../run_capture_airflowctl_help.py | 2 +- .../in_container/run_migration_reference.py | 2 +- scripts/tools/setup_breeze | 2 +- shared/README.md | 8 +- task-sdk/dev/generate_task_sdk_models.py | 2 +- 305 files changed, 1611 insertions(+), 3102 deletions(-) rename .github/actions/{install-pre-commit => install-prek}/action.yml (52%) delete mode 100644 dev/breeze/doc/images/output_static-checks.svg delete mode 100644 dev/breeze/doc/images/output_static-checks.txt delete mode 100644 dev/breeze/src/airflow_breeze/pre_commit_ids.py delete mode 100644 dev/breeze/src/airflow_breeze/pre_commit_ids_TEMPLATE.py.jinja2 delete mode 100755 scripts/ci/pre_commit/check_pre_commit_hooks.py rename scripts/ci/{pre_commit => prek}/.gitignore (100%) rename scripts/ci/{pre_commit => prek}/boring_cyborg.py (100%) rename scripts/ci/{pre_commit => prek}/breeze_cmd_line.py (96%) rename scripts/ci/{pre_commit => prek}/capture_airflowctl_help.py (91%) rename scripts/ci/{pre_commit => prek}/changelog_duplicates.py (100%) rename scripts/ci/{pre_commit => prek}/chart_schema.py (100%) rename scripts/ci/{pre_commit => prek}/check_aiobotocore_optional.py (100%) rename scripts/ci/{pre_commit => prek}/check_airflow_bug_report_template.py (95%) rename scripts/ci/{pre_commit => prek}/check_airflow_imports.py (96%) rename scripts/ci/{pre_commit => prek}/check_airflow_v_imports_in_tests.py (97%) rename scripts/ci/{pre_commit => prek}/check_base_operator_partial_arguments.py (97%) rename scripts/ci/{pre_commit => prek}/check_changelog_format.py (100%) rename scripts/ci/{pre_commit => prek}/check_common_sql_dependency.py (98%) rename scripts/ci/{pre_commit => prek}/check_default_configuration.py (90%) rename scripts/ci/{pre_commit => prek}/check_deferrable_default.py (100%) rename scripts/ci/{pre_commit => prek}/check_deprecations.py (100%) rename scripts/ci/{pre_commit => prek}/check_extra_packages_ref.py (91%) rename scripts/ci/{pre_commit => prek}/check_i18n_json.py (90%) rename scripts/ci/{pre_commit => prek}/check_imports_in_providers.py (91%) rename scripts/ci/{pre_commit => prek}/check_init_decorator_arguments.py (97%) rename scripts/ci/{pre_commit => prek}/check_init_in_tests.py (100%) rename scripts/ci/{pre_commit => prek}/check_integrations_list.py (98%) rename scripts/ci/{pre_commit => prek}/check_kubeconform.py (92%) rename scripts/ci/{pre_commit => prek}/check_lazy_logging.py (100%) rename scripts/ci/{pre_commit => prek}/check_license.py (97%) rename scripts/ci/{pre_commit => prek}/check_min_python_version.py (91%) rename scripts/ci/{pre_commit => prek}/check_order_dockerfile_extras.py (97%) rename scripts/ci/{pre_commit => prek}/check_provider_docs.py (99%) rename scripts/ci/{pre_commit => prek}/check_provider_version_compat.py (92%) rename scripts/ci/{pre_commit => prek}/check_provider_yaml_files.py (91%) rename scripts/ci/{pre_commit => prek}/check_providers_subpackages_all_have_init.py (95%) rename scripts/ci/{pre_commit => prek}/check_revision_heads_map.py (94%) rename scripts/ci/{pre_commit => prek}/check_sdk_imports.py (98%) rename scripts/ci/{pre_commit => prek}/check_shared_distributions_structure.py (98%) rename scripts/ci/{pre_commit => prek}/check_shared_distributions_usage.py (98%) rename scripts/ci/{pre_commit => prek}/check_system_tests.py (100%) rename scripts/ci/{pre_commit => prek}/check_system_tests_hidden_in_index.py (96%) rename scripts/ci/{pre_commit => prek}/check_template_context_variable_in_sync.py (98%) rename scripts/ci/{pre_commit => prek}/check_template_fields.py (91%) rename scripts/ci/{pre_commit => prek}/check_tests_in_right_folders.py (94%) rename scripts/ci/{pre_commit => prek}/check_ti_vs_tis_attributes.py (95%) rename scripts/ci/{pre_commit => prek}/checkout_no_credentials.py (100%) rename scripts/ci/{pre_commit/common_precommit_black_utils.py => prek/common_prek_black_utils.py} (93%) rename scripts/ci/{pre_commit/common_precommit_utils.py => prek/common_prek_utils.py} (97%) rename scripts/ci/{pre_commit => prek}/compile_provider_assets.py (94%) rename scripts/ci/{pre_commit => prek}/compile_ui_assets.py (92%) rename scripts/ci/{pre_commit => prek}/compile_ui_assets_dev.py (91%) rename scripts/ci/{pre_commit => prek}/decorator_operator_implements_custom_name.py (100%) rename scripts/ci/{pre_commit => prek}/diagram_hash.txt (100%) rename scripts/ci/{pre_commit => prek}/docstring_param_type.py (100%) rename scripts/ci/{pre_commit => prek}/draft7_schema.json (100%) rename scripts/ci/{pre_commit => prek}/generate_airflow_diagrams.py (96%) rename scripts/ci/{pre_commit => prek}/generate_openapi_spec.py (91%) rename scripts/ci/{pre_commit => prek}/generate_openapi_spec_fab.py (91%) rename scripts/ci/{pre_commit => prek}/generate_openapi_spec_keycloak.py (91%) rename scripts/ci/{pre_commit => prek}/generate_pypi_readme.py (94%) rename scripts/ci/{pre_commit => prek}/generate_volumes_for_sources.py (89%) rename scripts/ci/{pre_commit => prek}/inline_scripts_in_docker.py (97%) rename scripts/ci/{pre_commit => prek}/lint_dockerfile.py (96%) rename scripts/ci/{pre_commit => prek}/lint_helm.py (91%) rename scripts/ci/{pre_commit => prek}/lint_json_schema.py (98%) rename scripts/ci/{pre_commit => prek}/local_yml_mounts.py (90%) rename scripts/ci/{pre_commit => prek}/migration_reference.py (90%) rename scripts/ci/{pre_commit => prek}/mypy.py (95%) rename scripts/ci/{pre_commit => prek}/mypy_folder.py (96%) rename scripts/ci/{pre_commit => prek}/new_session_in_provide_session.py (100%) rename scripts/ci/{pre_commit => prek}/newsfragments.py (100%) rename scripts/ci/{pre_commit => prek}/prevent_deprecated_sqlalchemy_usage.py (100%) rename scripts/ci/{pre_commit => prek}/replace_bad_characters.py (100%) rename scripts/ci/{pre_commit => prek}/ruff_format.py (100%) rename scripts/ci/{pre_commit => prek}/significant_newsfragments_checker.py (100%) rename scripts/ci/{pre_commit => prek}/sort_in_the_wild.py (98%) rename scripts/ci/{pre_commit => prek}/sort_installed_providers.py (96%) rename scripts/ci/{pre_commit => prek}/sort_spelling_wordlist.py (96%) rename scripts/ci/{pre_commit => prek}/supported_versions.py (100%) rename scripts/ci/{pre_commit => prek}/ts_compile_lint_simple_auth_manager_ui.py (97%) rename scripts/ci/{pre_commit => prek}/ts_compile_lint_ui.py (97%) rename scripts/ci/{pre_commit => prek}/unittest_testcase.py (100%) rename scripts/ci/{pre_commit => prek}/update_airflow_pyproject_toml.py (98%) rename scripts/ci/{pre_commit => prek}/update_black_version.py (87%) rename scripts/ci/{pre_commit => prek}/update_breeze_config_hash.py (95%) rename scripts/ci/{pre_commit => prek}/update_chart_dependencies.py (96%) rename scripts/ci/{pre_commit => prek}/update_er_diagram.py (89%) rename scripts/ci/{pre_commit => prek}/update_example_dags_paths.py (96%) rename scripts/ci/{pre_commit => prek}/update_providers_build_files.py (96%) rename scripts/ci/{pre_commit => prek}/update_providers_dependencies.py (99%) rename scripts/ci/{pre_commit => prek}/update_source_date_epoch.py (96%) rename scripts/ci/{pre_commit => prek}/update_versions.py (95%) rename scripts/ci/{pre_commit/update_installers_and_pre_commit.py => prek/upgrade_important_versions.py} (83%) rename scripts/ci/{pre_commit => prek}/validate_chart_annotations.py (97%) rename scripts/ci/{pre_commit => prek}/validate_operators_init.py (100%) rename scripts/ci/{pre_commit => prek}/vendor_k8s_json_schema.py (100%) diff --git a/.github/.pre-commit-config.yaml b/.github/.pre-commit-config.yaml index 909f0c1cdca3c..69bb3b8c3c8df 100644 --- a/.github/.pre-commit-config.yaml +++ b/.github/.pre-commit-config.yaml @@ -18,7 +18,7 @@ default_stages: [manual] default_language_version: python: python311 -minimum_pre_commit_version: '3.2.0' +minimum_prek_version: '0.0.26' repos: - repo: https://github.com/eclipse-csi/octopin rev: 21360742e352e87450f99e180fdfc2cf774a72a3 diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-prek/action.yml similarity index 52% rename from .github/actions/install-pre-commit/action.yml rename to .github/actions/install-prek/action.yml index f2ee5868aa96c..5a92235d2f8b8 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-prek/action.yml @@ -16,38 +16,33 @@ # under the License. # --- -name: 'Install pre-commit' -description: 'Installs pre-commit and related packages' +name: 'Install prek' +description: 'Installs prek and related packages' inputs: python-version: description: 'Python version to use' default: "3.10" uv-version: description: 'uv version to use' - default: "0.8.9" # Keep this comment to allow automatic replacement of uv version - pre-commit-version: - description: 'pre-commit version to use' - default: "4.3.0" # Keep this comment to allow automatic replacement of pre-commit version - pre-commit-uv-version: - description: 'pre-commit-uv version to use' - default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version - skip-pre-commits: - description: "Skip some pre-commits from installation" + default: "0.8.11" # Keep this comment to allow automatic replacement of uv version + prek-version: + description: 'prek version to use' + default: "0.0.26" # Keep this comment to allow automatic replacement of prek version + skip-prek-hooks: + description: "Skip some prek hooks from installation" default: "" runs: using: "composite" steps: - - name: Install pre-commit, uv, and pre-commit-uv + - name: Install prek and uv shell: bash env: UV_VERSION: ${{inputs.uv-version}} - PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}} - PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}} - SKIP: ${{ inputs.skip-pre-commits }} + PREK_VERSION: ${{inputs.prek-version}} + SKIP: ${{ inputs.skip-prek-hooks }} run: | pip install uv==${UV_VERSION} || true - uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \ - --with pre-commit-uv==${PRE_COMMIT_UV_VERSION} + uv tool install prek==${PREK_VERSION} --with uv==${UV_VERSION} working-directory: ${{ github.workspace }} # We need to use tar file with archive to restore all the permissions and symlinks - name: "Delete ~.cache" @@ -59,36 +54,36 @@ runs: rm -rf ~/.cache echo shell: bash - - name: "Restore pre-commit cache" + - name: "Restore prek cache" uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: - key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-prek-v5-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} path: /tmp/ - id: restore-pre-commit-cache - - name: "Check if pre-commit cache tarball exists" + id: restore-prek-cache + - name: "Check if prek cache tarball exists" shell: bash run: | - if [ -f /tmp/cache-pre-commit.tar.gz ]; then - echo "✅ Cache tarball found: /tmp/cache-pre-commit.tar.gz" + if [ -f /tmp/cache-prek.tar.gz ]; then + echo "✅ Cache tarball found: /tmp/cache-prek.tar.gz" else - echo "❌ Cache tarball missing. Expected /tmp/cache-pre-commit.tar.gz" + echo "❌ Cache tarball missing. Expected /tmp/cache-prek.tar.gz" exit 1 fi - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + if: steps.restore-prek-cache.outputs.stash-hit == 'true' - name: "Restore .cache from the tar file" - run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz + run: tar -C ~ -xzf /tmp/cache-prek.tar.gz shell: bash - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + if: steps.restore-prek-cache.outputs.stash-hit == 'true' - name: "Show restored files" run: | echo "Restored files" du ~/ --max-depth=2 echo shell: bash - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' - - name: Install pre-commit hooks + if: steps.restore-prek-cache.outputs.stash-hit == 'true' + - name: Install prek hooks shell: bash - run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1) + run: prek install-hooks || (cat ~/.cache/prek/prek.log && exit 1) working-directory: ${{ github.workspace }} env: - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 1bfdfafef6c41..351ad3c98e88a 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -519,8 +519,8 @@ firstPRWelcomeComment: > Here are some useful points: - - Pay attention to the quality of your code (ruff, mypy and type annotations). Our [pre-commits]( - https://github.com/apache/airflow/blob/main/contributing-docs/08_static_code_checks.rst#prerequisites-for-pre-commit-hooks) + - Pay attention to the quality of your code (ruff, mypy and type annotations). Our [prek-hooks]( + https://github.com/apache/airflow/blob/main/contributing-docs/08_static_code_checks.rst#prerequisites-for-prek-hooks) will help you with that. - In case of a new feature add useful documentation (in docstrings or in `docs/` directory). diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index f3982389301e6..d289f38d1a358 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -48,8 +48,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string docker-cache: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index b76eb38422dfd..32b7321cc601b 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -40,8 +40,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to run only basic checks (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string default-python-version: @@ -150,13 +150,13 @@ jobs: retention-days: '2' if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true' - install-pre-commit: + install-prek: timeout-minutes: 5 - name: "Install pre-commit for cache" + name: "Install prek for cache" runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} steps: - name: "Cleanup repo" shell: bash @@ -168,21 +168,21 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - skip-pre-commits: ${{ inputs.skip-pre-commits }} + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building - # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host + # The CI Docker image and they can be run entirely using the prek virtual environments on host static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -197,12 +197,12 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{ steps.breeze.outputs.host-python-version }} - skip-pre-commits: ${{ inputs.skip-pre-commits }} + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -211,12 +211,12 @@ jobs: persist-credentials: false - name: "Static checks: basic checks only" run: > - breeze static-checks --show-diff-on-failure --color always --initialize-environment - --skip-image-upgrade-check --commit-ref "${{ github.sha }}" + prek --show-diff-on-failure --color always + --from-ref "${{ github.sha }}" --to-ref "${{ github.sha }}" env: VERBOSE: "false" - SKIP_BREEZE_PRE_COMMITS: "true" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP_BREEZE_PREK_HOOKS: "true" + SKIP: ${{ inputs.skip-prek-hooks }} COLUMNS: "202" test-git-clone-on-windows: @@ -234,7 +234,7 @@ jobs: timeout-minutes: 45 name: "Upgrade checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' @@ -251,28 +251,28 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - skip-pre-commits: ${{ inputs.skip-pre-commits }} - - name: "Autoupdate all pre-commits" - run: pre-commit autoupdate --freeze - - name: "Autoupdate Lucas-C/pre-commit-hooks to bleeding edge" - run: pre-commit autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} + - name: "Autoupdate all prek hooks" + run: prek autoupdate --freeze + - name: "Autoupdate Lucas-C hooks to bleeding edge" + run: prek autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks - name: "Check if there are any changes in pre-commit hooks" run: git diff --exit-code - name: "Run automated upgrade for black" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose --hook-stage manual update-black-version if: always() - name: "Run automated upgrade for chart dependencies" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose --hook-stage manual update-chart-dependencies @@ -282,30 +282,30 @@ jobs: # get notified about it - until it stabilizes in 1.* version - name: "Run automated upgrade for uv (not failing - just informational)" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers-and-pre-commit || true + --hook-stage manual upgrade-important-versions || true if: always() env: UPGRADE_UV: "true" UPGRADE_PYTHON: "false" UPGRADE_GOLANG: "true" UPGRADE_PIP: "false" - UPGRADE_PRE_COMMIT: "false" + UPGRADE_PREK: "false" UPGRADE_NODE_LTS: "false" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Run automated upgrade for pip, pre-commit and node (failing if needed)" + - name: "Run automated upgrade for important versions minus uv(failing if needed)" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers-and-pre-commit + --hook-stage manual upgrade-important-versions if: always() env: UPGRADE_UV: "false" UPGRADE_PYTHON: "true" UPGRADE_GOLANG: "false" UPGRADE_PIP: "true" - UPGRADE_PRE_COMMIT: "true" + UPGRADE_PREK: "true" UPGRADE_NODE_LTS: "true" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 13d45625c35dc..fa500f435c4a4 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -122,7 +122,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} - skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} source-head-ref: ${{ steps.source-run-info.outputs.head-ref }} @@ -168,8 +168,8 @@ jobs: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} - run-pin-versions-pre-commit: - name: "Run pin-versions pre-commit" + run-pin-versions-prek: + name: "Run pin-versions prek" needs: [build-info] runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} steps: @@ -177,24 +177,22 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - name: "Install Python 3.11 as 3.11+ is needed by pin-versions pre-commit" + - name: "Install Python 3.11 as 3.11+ is needed by pin-versions" uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 with: python-version: 3.11 cache: "pip" - - name: Install pre-commit, uv, and pre-commit-uv + - name: Install prek, uv shell: bash env: - UV_VERSION: "0.8.9" # Keep this comment to allow automatic replacement of uv version - PRE_COMMIT_VERSION: "4.3.0" # Keep this comment to allow automatic replacement of pre-commit version - PRE_COMMIT_UV_VERSION: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version + UV_VERSION: "0.8.11" # Keep this comment to allow automatic replacement of uv version + PREK_VERSION: "0.0.26" # Keep this comment to allow automatic replacement of prek version run: | pip install uv==${UV_VERSION} || true - uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \ - --with pre-commit-uv==${PRE_COMMIT_UV_VERSION} - - name: "Run pin-versions pre-commit" + uv tool install prek==${PREK_VERSION} --with uv==${UV_VERSION} + - name: "Run pin-versions" run: > - pre-commit run -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual + prek -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual pin-versions env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -210,7 +208,7 @@ jobs: needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} @@ -255,7 +253,7 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -301,7 +299,7 @@ jobs: latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 89014e80f6742..677423cd71257 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -115,7 +115,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} - skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} @@ -171,7 +171,7 @@ jobs: needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} @@ -216,7 +216,7 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 7fd13495aa827..2d52aec1ad04f 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -72,8 +72,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to build CI images (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string include-success-outputs: @@ -122,9 +122,9 @@ on: # yamllint disable-line rule:truthy permissions: contents: read jobs: - install-pre-commit: + install-prek: timeout-minutes: 5 - name: "Install pre-commit for cache (only canary runs)" + name: "Install prek for cache (only canary runs)" runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -143,23 +143,23 @@ jobs: uses: ./.github/actions/breeze id: breeze if: inputs.canary-run == 'true' - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} if: inputs.canary-run == 'true' - - name: "Prepare .tar file from pre-commit cache" + - name: "Prepare .tar file from prek cache" run: | - tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv + tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek .cache/uv shell: bash if: inputs.canary-run == 'true' - - name: "Save pre-commit cache" + - name: "Save prek cache" uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: # yamllint disable rule:line-length - key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} - path: /tmp/cache-pre-commit.tar.gz + key: cache-prek-v5-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-prek.tar.gz if-no-files-found: 'error' retention-days: '2' if: inputs.canary-run == 'true' @@ -168,7 +168,7 @@ jobs: timeout-minutes: 45 name: "Static checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -189,16 +189,16 @@ jobs: python: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - name: "Static checks" - run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment + run: prek --all-files --show-diff-on-failure --color always env: VERBOSE: "false" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} COLUMNS: "202" SKIP_GROUP_OUTPUT: "true" DEFAULT_BRANCH: ${{ inputs.branch }} @@ -208,7 +208,7 @@ jobs: timeout-minutes: 45 name: "MyPy checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek if: inputs.needs-mypy == 'true' strategy: fail-fast: false @@ -232,13 +232,13 @@ jobs: python: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - name: "MyPy checks for ${{ matrix.mypy-check }}" - run: pre-commit run --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files + run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files env: VERBOSE: "false" COLUMNS: "202" diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml index 04e308c306138..45efd3364ea6d 100644 --- a/.github/workflows/news-fragment.yml +++ b/.github/workflows/news-fragment.yml @@ -60,4 +60,4 @@ jobs: env: BASE_REF: ${{ github.base_ref }} run: > - uv run scripts/ci/pre_commit/significant_newsfragments_checker.py + uv run scripts/ci/prek/significant_newsfragments_checker.py diff --git a/.gitpod.yml b/.gitpod.yml index fa77c07eda214..2a724753de4ec 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -24,13 +24,13 @@ image: gitpod/workspace-python-3.11 tasks: - init: ./scripts/ci/install_breeze.sh - - name: Install pre-commit + - name: Install prek openMode: split-right command: | printf '%s\n' "export PIP_USER=no" >> "$HOME/.bashrc" source "$HOME/.bashrc" - pip install pre-commit - pre-commit install + pip install prek + prek install echo "for running integration test with breeze" # Ports to expose on workspace startup diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eec81cf26e27a..6831caa7d6a1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ default_language_version: python: python3 node: 22.18.0 golang: 1.24.0 -minimum_pre_commit_version: '3.2.0' +minimum_prek_version: '0.0.26' exclude: ^.*/.*_vendor/ repos: - repo: meta @@ -188,44 +188,44 @@ repos: hooks: - id: check-min-python-version name: Check minimum Python version - entry: ./scripts/ci/pre_commit/check_min_python_version.py + entry: ./scripts/ci/prek/check_min_python_version.py language: python additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: update-black-version name: Update black versions everywhere (manual) - entry: ./scripts/ci/pre_commit/update_black_version.py + entry: ./scripts/ci/prek/update_black_version.py stages: ['manual'] language: python files: ^\.pre-commit-config\.yaml$ additional_dependencies: ['pyyaml>=6.0.2'] pass_filenames: false require_serial: true - - id: update-installers-and-pre-commit - name: Update installers and pre-commit to latest (manual) - entry: ./scripts/ci/pre_commit/update_installers_and_pre_commit.py + - id: upgrade-important-versions + name: Upgrade important versions (manual) + entry: ./scripts/ci/prek/upgrade_important_versions.py stages: ['manual'] language: python files: > (?x) ^\.pre-commit-config\.yaml$| ^\.github/\.pre-commit-config\.yaml$| - ^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$ + ^scripts/ci/prek/update_installers_and_prek\.py$ pass_filenames: false require_serial: true additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=25"] - id: update-chart-dependencies name: Update chart dependencies to latest (manual) - entry: ./scripts/ci/pre_commit/update_chart_dependencies.py + entry: ./scripts/ci/prek/update_chart_dependencies.py stages: ['manual'] language: python additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0'] - files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_build_dependencies\.py$ + files: ^\.pre-commit-config\.yaml$|^scripts/ci/prek/update_build_dependencies\.py$ pass_filenames: false require_serial: true - id: check-taskinstance-tis-attrs name: Check that TI and TIS have the same attributes - entry: ./scripts/ci/pre_commit/check_ti_vs_tis_attributes.py + entry: ./scripts/ci/prek/check_ti_vs_tis_attributes.py language: python additional_dependencies: ['rich>=12.4.4'] files: ^airflow-core/src/airflow/models/taskinstance\.py$|^airflow-core/src/airflow/models/taskinstancehistory\.py$ @@ -234,7 +234,7 @@ repos: - id: check-deferrable-default name: Check and fix default value of default_deferrable language: python - entry: ./scripts/ci/pre_commit/check_deferrable_default.py + entry: ./scripts/ci/prek/check_deferrable_default.py pass_filenames: false # libcst doesn't have source wheels for all PY except PY3.12, excluding it additional_dependencies: ['libcst>=1.8.1'] @@ -370,30 +370,30 @@ repos: entry: zizmor - repo: local # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because - # we try to minimise the number of passes that must happen in order to apply some of the changes - # done by pre-commits. Some of the pre-commits not only check for errors but also fix them. This means - # that output from an earlier pre-commit becomes input to another pre-commit. Splitting the local - # scripts of our and adding some other non-local pre-commit in-between allows us to handle such + # we try to minimize the number of passes that must happen to apply some of the changes + # done by prek-hooks. Some of the prek hooks not only check for errors but also fix them. This means + # that output from an earlier prek hook becomes input to another prek hook. Splitting the local + # scripts of our and adding some other non-local prek hook in-between allows us to handle such # changes quickly - especially when we want the early modifications from the first local group - # to be applied before the non-local pre-commits are run + # to be applied before the non-local prek hooks are run hooks: - id: update-providers-dependencies name: Update dependencies for providers - entry: ./scripts/ci/pre_commit/update_providers_dependencies.py + entry: ./scripts/ci/prek/update_providers_dependencies.py language: python always_run: true pass_filenames: false additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2', 'tomli>=2.0.1'] - id: check-shared-distributions-structure name: Check shared distributions structure - entry: ./scripts/ci/pre_commit/check_shared_distributions_structure.py + entry: ./scripts/ci/prek/check_shared_distributions_structure.py language: python additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1'] pass_filenames: false files: ^shared/.*$ - id: check-shared-distributions-usage name: Check shared distributions usage - entry: ./scripts/ci/pre_commit/check_shared_distributions_usage.py + entry: ./scripts/ci/prek/check_shared_distributions_usage.py language: python additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1'] pass_filenames: false @@ -402,13 +402,13 @@ repos: name: No templated field logic checks in operator __init__ description: Prevent templated field logic checks in operators' __init__ language: python - entry: ./scripts/ci/pre_commit/validate_operators_init.py + entry: ./scripts/ci/prek/validate_operators_init.py pass_filenames: true files: ^providers/.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ additional_dependencies: ['rich>=12.4.4'] - id: update-providers-build-files name: Update providers build files - entry: ./scripts/ci/pre_commit/update_providers_build_files.py + entry: ./scripts/ci/prek/update_providers_build_files.py language: python pass_filenames: true files: | @@ -425,7 +425,7 @@ repos: - id: check-airflow-v-imports-in-tests name: Check AIRFLOW_V imports in tests language: python - entry: ./scripts/ci/pre_commit/check_airflow_v_imports_in_tests.py + entry: ./scripts/ci/prek/check_airflow_v_imports_in_tests.py pass_filenames: true files: ^providers/.*/tests/.+\.py$ additional_dependencies: ['rich>=12.4.4'] @@ -442,7 +442,7 @@ repos: - id: ruff-format name: Run 'ruff format' description: "Run 'ruff format' for extremely fast Python formatting" - entry: ./scripts/ci/pre_commit/ruff_format.py + entry: ./scripts/ci/prek/ruff_format.py language: python types_or: [python, pyi] args: [] @@ -451,7 +451,7 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$ - id: replace-bad-characters name: Replace bad characters - entry: ./scripts/ci/pre_commit/replace_bad_characters.py + entry: ./scripts/ci/prek/replace_bad_characters.py language: python types: [file, text] exclude: ^clients/gen/go\.sh$|^\.gitmodules$|^airflow-core/src/airflow/ui/openapi-gen/|.*/dist/.*|\.go$|/go\.(mod|sum)$ @@ -459,7 +459,7 @@ repos: - id: lint-dockerfile name: Lint Dockerfile language: python - entry: ./scripts/ci/pre_commit/lint_dockerfile.py + entry: ./scripts/ci/prek/lint_dockerfile.py files: Dockerfile.*$ pass_filenames: true require_serial: true @@ -469,7 +469,7 @@ repos: files: ^airflow-core/src/airflow/.*\.py$ require_serial: true exclude: ^airflow-core/src/airflow/kubernetes/ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.kubernetes' --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." additional_dependencies: ['rich>=12.4.4'] @@ -482,7 +482,7 @@ repos: (?x) ^airflow-core/src/airflow/openlineage/| ^airflow/providers/common/compat/openlineage/facet.py$ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^openlineage\.client\.(facet|run)' --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." additional_dependencies: ['rich>=12.4.4'] @@ -491,7 +491,7 @@ repos: language: python files: ^\.github/ISSUE_TEMPLATE/3-airflow_providers_bug_report\.yml$ require_serial: true - entry: ./scripts/ci/pre_commit/check_airflow_bug_report_template.py + entry: ./scripts/ci/prek/check_airflow_bug_report_template.py additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-cncf-k8s-only-for-executors name: Check cncf.kubernetes imports used for executors only @@ -506,13 +506,13 @@ repos: ^airflow-core/src/airflow/serialization/serialized_objects\.py$| ^airflow-core/src/airflow/serialization/serializers/kubernetes\.py$| ^airflow-core/src/airflow/utils/sqlalchemy\.py$ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.providers\.cncf\.kubernetes' --message "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." additional_dependencies: ['rich>=12.4.4'] - id: update-local-yml-file name: Update mounts in the local yml file - entry: ./scripts/ci/pre_commit/local_yml_mounts.py + entry: ./scripts/ci/prek/local_yml_mounts.py language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false @@ -520,7 +520,7 @@ repos: - id: check-sql-dependency-common-data-structure name: Check dependency of SQL providers description: Check dependency of SQL Providers with common data structure - entry: ./scripts/ci/pre_commit/check_common_sql_dependency.py + entry: ./scripts/ci/prek/check_common_sql_dependency.py language: python files: ^providers/.*/src/airflow/providers/.*/hooks/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2', 'packaging>=25'] @@ -530,18 +530,18 @@ repos: language: python files: ^airflow-core/docs/extra-packages-ref\.rst$|^hatch_build\.py$ pass_filenames: false - entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py + entry: ./scripts/ci/prek/check_extra_packages_ref.py additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0', 'tabulate>=0.9.0'] - id: check-extras-order name: Check order of extras in Dockerfile - entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py + entry: ./scripts/ci/prek/check_order_dockerfile_extras.py language: python files: ^Dockerfile$ pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: generate-airflow-diagrams name: Generate airflow diagrams - entry: ./scripts/ci/pre_commit/generate_airflow_diagrams.py + entry: ./scripts/ci/prek/generate_airflow_diagrams.py language: python files: > (?x) @@ -551,7 +551,7 @@ repos: additional_dependencies: ['rich>=12.4.4', 'diagrams>=0.23.4'] - id: generate-volumes-for-sources name: Generate volumes for docker compose - entry: ./scripts/ci/pre_commit/generate_volumes_for_sources.py + entry: ./scripts/ci/prek/generate_volumes_for_sources.py language: python files: ^providers/.*/provider\.yaml$ pass_filenames: false @@ -559,7 +559,7 @@ repos: additional_dependencies: ['rich>=12.4.4'] - id: prevent-deprecated-sqlalchemy-usage name: Prevent deprecated sqlalchemy usage - entry: ./scripts/ci/pre_commit/prevent_deprecated_sqlalchemy_usage.py + entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py language: python additional_dependencies: ['rich>=12.4.4'] files: > @@ -569,19 +569,19 @@ repos: pass_filenames: true - id: update-supported-versions name: Updates supported versions in documentation - entry: ./scripts/ci/pre_commit/supported_versions.py + entry: ./scripts/ci/prek/supported_versions.py language: python - files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$ + files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$ pass_filenames: false additional_dependencies: ['tabulate>=0.9.0'] - id: check-revision-heads-map name: Check that the REVISION_HEADS_MAP is up-to-date language: python - entry: ./scripts/ci/pre_commit/check_revision_heads_map.py + entry: ./scripts/ci/prek/check_revision_heads_map.py pass_filenames: false files: > (?x) - ^scripts/ci/pre_commit/version_heads_map\.py$| + ^scripts/ci/prek/version_heads_map\.py$| ^airflow-core/src/airflow/migrations/versions/.*$|^airflow-core/src/airflow/migrations/versions| ^providers/fab/src/airflow/providers/fab/migrations/versions/.*$|^providers/fab/src/airflow/providers/fab/migrations/versions| ^airflow-core/src/airflow/utils/db\.py$| @@ -589,7 +589,7 @@ repos: additional_dependencies: ['packaging>=25', 'rich>=12.4.4'] - id: update-version name: Update versions in docs - entry: ./scripts/ci/pre_commit/update_versions.py + entry: ./scripts/ci/prek/update_versions.py language: python files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$ pass_filenames: false @@ -695,7 +695,7 @@ repos: - id: check-only-new-session-with-provide-session name: Check NEW_SESSION is only used with @provide_session language: python - entry: ./scripts/ci/pre_commit/new_session_in_provide_session.py + entry: ./scripts/ci/prek/new_session_in_provide_session.py pass_filenames: true files: ^airflow-core/src/airflow/.+\.py$ exclude: ^airflow-core/src/airflow/serialization/pydantic/.* @@ -772,7 +772,7 @@ repos: ^devel-common/src/sphinx_exts/removemarktransform\.py| ^devel-common/src/tests_common/test_utils/db\.py| ^airflow-core/newsfragments/41761.significant\.rst$| - ^scripts/ci/pre_commit/vendor_k8s_json_schema\.py$| + ^scripts/ci/prek/vendor_k8s_json_schema\.py$| ^scripts/ci/docker-compose/integration-keycloak\.yml$| ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$| ^airflow-core/tests/| @@ -792,21 +792,21 @@ repos: - id: check-base-operator-partial-arguments name: Check BaseOperator and partial() arguments language: python - entry: ./scripts/ci/pre_commit/check_base_operator_partial_arguments.py + entry: ./scripts/ci/prek/check_base_operator_partial_arguments.py pass_filenames: false files: ^airflow-core/src/airflow/models/(?:base|mapped)operator\.py$ additional_dependencies: ['rich>=12.4.4'] - id: check-init-decorator-arguments name: Sync model __init__ and decorator arguments language: python - entry: ./scripts/ci/pre_commit/check_init_decorator_arguments.py + entry: ./scripts/ci/prek/check_init_decorator_arguments.py pass_filenames: false files: ^task-sdk/src/airflow/sdk/definitions/dag\.py$|^task-sdk/src/airflow/sdk/definitions/decorators/task_group\.py$ additional_dependencies: ['rich>=12.4.4'] - id: check-template-context-variable-in-sync name: Sync template context variable refs language: python - entry: ./scripts/ci/pre_commit/check_template_context_variable_in_sync.py + entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py files: ^airflow-core/src/airflow/models/taskinstance\.py$|^task-sdk/src/airflow/sdk/definitions/context\.py$|^airflow-core/docs/templates-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] - id: check-base-operator-usage @@ -851,7 +851,7 @@ repos: name: Check providers import hook lineage code from compat description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of airflow.lineage.hook. - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.lineage\.hook' --message "Only TYPE_CHECKING imports from `airflow.lineage.hook` are allowed in providers." --only_top_level @@ -861,7 +861,7 @@ repos: - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name language: python - entry: ./scripts/ci/pre_commit/decorator_operator_implements_custom_name.py + entry: ./scripts/ci/prek/decorator_operator_implements_custom_name.py pass_filenames: true files: ^airflow-core/src/airflow/.*\.py$ - id: check-core-deprecation-classes @@ -892,13 +892,13 @@ repos: pass_filenames: true - id: check-apache-license-rat name: Check if licenses are OK for Apache - entry: ./scripts/ci/pre_commit/check_license.py + entry: ./scripts/ci/prek/check_license.py language: python files: ^LICENSE$ pass_filenames: false - id: check-aiobotocore-optional name: Check if aiobotocore is an optional dependency only - entry: ./scripts/ci/pre_commit/check_aiobotocore_optional.py + entry: ./scripts/ci/prek/check_aiobotocore_optional.py language: python files: ^providers/.*/provider\.yaml$ pass_filenames: true @@ -907,34 +907,34 @@ repos: - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency language: python - entry: ./scripts/ci/pre_commit/boring_cyborg.py + entry: ./scripts/ci/prek/boring_cyborg.py pass_filenames: false require_serial: true additional_dependencies: ['pyyaml>=6.0.2', 'termcolor==2.5.0', 'wcmatch==8.2'] - id: update-in-the-wild-to-be-sorted name: Sort INTHEWILD.md alphabetically - entry: ./scripts/ci/pre_commit/sort_in_the_wild.py + entry: ./scripts/ci/prek/sort_in_the_wild.py language: python files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ pass_filenames: false require_serial: true - id: update-installed-providers-to-be-sorted name: Sort and uniquify installed_providers.txt - entry: ./scripts/ci/pre_commit/sort_installed_providers.py + entry: ./scripts/ci/prek/sort_installed_providers.py language: python files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$ pass_filenames: false require_serial: true - id: update-spelling-wordlist-to-be-sorted name: Sort spelling_wordlist.txt - entry: ./scripts/ci/pre_commit/sort_spelling_wordlist.py + entry: ./scripts/ci/prek/sort_spelling_wordlist.py language: python files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ require_serial: true pass_filenames: false - id: lint-helm-chart name: Lint Helm Chart - entry: ./scripts/ci/pre_commit/lint_helm.py + entry: ./scripts/ci/prek/lint_helm.py language: python pass_filenames: false files: ^chart @@ -942,14 +942,14 @@ repos: additional_dependencies: ['rich>=12.4.4','requests>=2.31.0'] - id: validate-chart-annotations name: Validate chart annotations - entry: ./scripts/ci/pre_commit/validate_chart_annotations.py + entry: ./scripts/ci/prek/validate_chart_annotations.py language: python pass_filenames: false files: ^chart/Chart\.yaml$ additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4'] - id: kubeconform name: Kubeconform check on our helm chart - entry: ./scripts/ci/pre_commit/check_kubeconform.py + entry: ./scripts/ci/prek/check_kubeconform.py language: python pass_filenames: false files: ^chart @@ -967,14 +967,14 @@ repos: stages: ['manual'] types_or: [javascript, ts, tsx] files: ^airflow-core/src/airflow/ui/|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/ - entry: ./scripts/ci/pre_commit/compile_ui_assets.py + entry: ./scripts/ci/prek/compile_ui_assets.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - id: compile-fab-assets name: Compile FAB provider assets language: node files: ^providers/fab/.*/www/ - entry: ./scripts/ci/pre_commit/compile_provider_assets.py fab + entry: ./scripts/ci/prek/compile_provider_assets.py fab pass_filenames: false additional_dependencies: ['yarn@1.22.21'] - id: compile-ui-assets-dev @@ -983,30 +983,20 @@ repos: stages: ['manual'] types_or: [javascript, ts, tsx] files: ^airflow-core/src/airflow/ui/|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/ - entry: ./scripts/ci/pre_commit/compile_ui_assets_dev.py + entry: ./scripts/ci/prek/compile_ui_assets_dev.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - id: check-providers-subpackages-init-file-exist name: Provider subpackage init files are there pass_filenames: false always_run: true - entry: ./scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py + entry: ./scripts/ci/prek/check_providers_subpackages_all_have_init.py language: python require_serial: true additional_dependencies: ['rich>=12.4.4'] - - id: check-pre-commit-information-consistent - name: Validate hook IDs & names and sync with docs - entry: ./scripts/ci/pre_commit/check_pre_commit_hooks.py - args: - - --max-length=53 - language: python - files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/check_pre_commit_hooks\.py$ - additional_dependencies: ['pyyaml>=6.0.2', 'jinja2', 'black==24.10.0', 'tabulate>=0.9.0', 'rich>=12.4.4'] - require_serial: true - pass_filenames: false - id: check-integrations-list-consistent name: Sync integrations list with docs - entry: ./scripts/ci/pre_commit/check_integrations_list.py + entry: ./scripts/ci/prek/check_integrations_list.py language: python files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$ additional_dependencies: ['black==24.10.0', 'tabulate>=0.9.0', 'rich>=12.4.4', 'pyyaml>=6.0.2'] @@ -1015,7 +1005,7 @@ repos: - id: update-breeze-readme-config-hash name: Update Breeze README.md with config files hash language: python - entry: ./scripts/ci/pre_commit/update_breeze_config_hash.py + entry: ./scripts/ci/prek/update_breeze_config_hash.py files: ^dev/breeze/pyproject\.toml$|^dev/breeze/README\.md$ pass_filenames: false require_serial: true @@ -1023,11 +1013,11 @@ repos: - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml language: python - entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py + entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py files: > (?x) ^.*/pyproject\.toml$| - ^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$| + ^scripts/ci/prek/update_airflow_pyproject_toml\.py$| ^providers/.*/pyproject\.toml$| ^providers/.*/provider\.yaml$ pass_filenames: false @@ -1036,7 +1026,7 @@ repos: - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds language: python - entry: ./scripts/ci/pre_commit/update_source_date_epoch.py + entry: ./scripts/ci/prek/update_source_date_epoch.py files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] pass_filenames: false @@ -1052,7 +1042,7 @@ repos: additional_dependencies: ['click>=8.1.8', 'rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-tests-in-the-right-folders name: Check if tests are in the right folders - entry: ./scripts/ci/pre_commit/check_tests_in_right_folders.py + entry: ./scripts/ci/prek/check_tests_in_right_folders.py language: python files: ^airflow-core/tests/.*\.py$ pass_filenames: true @@ -1060,14 +1050,14 @@ repos: additional_dependencies: ['rich>=12.4.4'] - id: check-system-tests-present name: Check if system tests have required segments of code - entry: ./scripts/ci/pre_commit/check_system_tests.py + entry: ./scripts/ci/prek/check_system_tests.py language: python files: ^.*/tests/system/.*/example_[^/]*\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme name: Generate PyPI README - entry: ./scripts/ci/pre_commit/generate_pypi_readme.py + entry: ./scripts/ci/prek/generate_pypi_readme.py language: python files: ^README\.md$ pass_filenames: false @@ -1081,10 +1071,10 @@ repos: additional_dependencies: ['markdownlint-cli@0.38.0'] - id: lint-json-schema name: Lint JSON Schema files - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - - scripts/ci/pre_commit/draft7_schema.json + - scripts/ci/prek/draft7_schema.json language: python pass_filenames: true files: .*\.schema\.json$ @@ -1092,7 +1082,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint NodePort Service - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url - https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json @@ -1103,7 +1093,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint Docker compose files - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url - https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json @@ -1119,7 +1109,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint chart/values.schema.json - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - chart/values_schema.schema.json @@ -1131,13 +1121,13 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: update-vendored-in-k8s-json-schema name: Vendor k8s definitions into values.schema.json - entry: ./scripts/ci/pre_commit/vendor_k8s_json_schema.py + entry: ./scripts/ci/prek/vendor_k8s_json_schema.py language: python files: ^chart/values\.schema\.json$ additional_dependencies: ['requests==2.32.3'] - id: lint-json-schema name: Lint chart/values.yaml - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --enforce-defaults - --spec-file @@ -1150,7 +1140,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint config_templates/config.yml - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - airflow-core/src/airflow/config_templates/config.yml.schema.json @@ -1162,14 +1152,14 @@ repos: - id: check-persist-credentials-disabled-in-github-workflows name: Check persistent creds in workflow files description: Check that workflow files have persist-credentials disabled - entry: ./scripts/ci/pre_commit/checkout_no_credentials.py + entry: ./scripts/ci/prek/checkout_no_credentials.py language: python pass_filenames: true files: ^\.github/workflows/.*\.yml$ additional_dependencies: ['pyyaml>=6.0.1', 'rich>=12.4.4'] - id: check-docstring-param-types name: Check that docstrings do not specify param types - entry: ./scripts/ci/pre_commit/docstring_param_type.py + entry: ./scripts/ci/prek/docstring_param_type.py language: python pass_filenames: true files: \.py$ @@ -1184,21 +1174,21 @@ repos: files: \.zip$ - id: check-code-deprecations name: Check deprecations categories in decorators - entry: ./scripts/ci/pre_commit/check_deprecations.py + entry: ./scripts/ci/prek/check_deprecations.py language: python pass_filenames: true files: ^airflow-core/src/airflow/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'python-dateutil>=2.9.0'] - id: lint-chart-schema name: Lint chart/values.schema.json file - entry: ./scripts/ci/pre_commit/chart_schema.py + entry: ./scripts/ci/prek/chart_schema.py language: python pass_filenames: false files: ^chart/values\.schema\.json$ require_serial: true - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts - entry: ./scripts/ci/pre_commit/inline_scripts_in_docker.py + entry: ./scripts/ci/prek/inline_scripts_in_docker.py language: python pass_filenames: false files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$ @@ -1207,19 +1197,19 @@ repos: name: Check changelogs for duplicate entries language: python files: changelog\.(rst|txt)$ - entry: ./scripts/ci/pre_commit/changelog_duplicates.py + entry: ./scripts/ci/prek/changelog_duplicates.py pass_filenames: true - id: check-changelog-format name: Check changelog format language: python files: changelog\.(rst|txt)$ - entry: ./scripts/ci/pre_commit/check_changelog_format.py + entry: ./scripts/ci/prek/check_changelog_format.py pass_filenames: true - id: check-newsfragments-are-valid name: Check newsfragments are valid language: python files: newsfragments/.*\.rst$ - entry: ./scripts/ci/pre_commit/newsfragments.py + entry: ./scripts/ci/prek/newsfragments.py pass_filenames: true # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes # This is fast, so not too much downside @@ -1229,7 +1219,7 @@ repos: # Significant newsfragments follows a special format so that we can group information easily. language: python files: ^airflow-core/newsfragments/.*\.rst$ - entry: ./scripts/ci/pre_commit/significant_newsfragments_checker.py + entry: ./scripts/ci/prek/significant_newsfragments_checker.py pass_filenames: false additional_dependencies: ['docutils>=0.21.2', 'pygments>=2.19.1', 'jinja2>=3.1.5'] # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes @@ -1238,20 +1228,20 @@ repos: - id: update-breeze-cmd-output name: Update breeze docs description: Update output of breeze commands in Breeze documentation - entry: ./scripts/ci/pre_commit/breeze_cmd_line.py + entry: ./scripts/ci/prek/breeze_cmd_line.py language: python files: > (?x) ^dev/breeze/.*$| ^\.pre-commit-config\.yaml$| - ^scripts/ci/pre_commit/breeze_cmd_line\.py$| + ^scripts/ci/prek/breeze_cmd_line\.py$| ^generated/provider_dependencies\.json$ require_serial: true pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: check-example-dags-urls name: Check that example dags url include provider versions - entry: ./scripts/ci/pre_commit/update_example_dags_paths.py + entry: ./scripts/ci/prek/update_example_dags_paths.py language: python pass_filenames: true files: ^airflow-core/docs/.*example-dags\.rst$|^docs/.*index\.rst$^airflow-core/docs/.*index\.rst$ @@ -1259,21 +1249,21 @@ repos: always_run: true - id: check-system-tests-tocs name: Check that system tests is properly added - entry: ./scripts/ci/pre_commit/check_system_tests_hidden_in_index.py + entry: ./scripts/ci/prek/check_system_tests_hidden_in_index.py language: python pass_filenames: true files: ^providers/.*/docs/index\.rst$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-lazy-logging name: Check that all logging methods are lazy - entry: ./scripts/ci/pre_commit/check_lazy_logging.py + entry: ./scripts/ci/prek/check_lazy_logging.py language: python pass_filenames: true files: \.py$ additional_dependencies: ['rich>=12.4.4', 'astor>=0.8.1'] - id: create-missing-init-py-files-tests name: Create missing init.py files in tests - entry: ./scripts/ci/pre_commit/check_init_in_tests.py + entry: ./scripts/ci/prek/check_init_in_tests.py language: python additional_dependencies: ['rich>=12.4.4'] pass_filenames: false @@ -1281,13 +1271,13 @@ repos: - id: check-tests-unittest-testcase name: Unit tests do not inherit from unittest.TestCase description: Check that unit tests do not inherit from unittest.TestCase - entry: ./scripts/ci/pre_commit/unittest_testcase.py + entry: ./scripts/ci/prek/unittest_testcase.py language: python pass_filenames: true files: ^airflow-core/tests/.*\.py$ - id: check-provider-docs-valid name: Validate provider doc files - entry: ./scripts/ci/pre_commit/check_provider_docs.py + entry: ./scripts/ci/prek/check_provider_docs.py language: python files: ^providers/.*/provider\.yaml$|^.*/docs/.* additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2', 'jinja2'] @@ -1395,7 +1385,7 @@ repos: (?x) ^airflow-core/src/airflow/ui/node-modules/.*| ^airflow-core/src/airflow/ui/.pnpm-store - entry: ./scripts/ci/pre_commit/ts_compile_lint_ui.py + entry: ./scripts/ci/prek/ts_compile_lint_ui.py additional_dependencies: ['pnpm@9.7.1'] pass_filenames: true require_serial: true @@ -1412,16 +1402,16 @@ repos: (?x) ^airflow-core/src/airflow/api_fastapi/node-modules/.*| ^airflow-core/src/airflow/api_fastapi/.pnpm-store - entry: ./scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py + entry: ./scripts/ci/prek/ts_compile_lint_simple_auth_manager_ui.py additional_dependencies: ['pnpm@9.7.1'] pass_filenames: true require_serial: true - ## ADD MOST PRE-COMMITS ABOVE THAT LINE - # The below pre-commits are those requiring CI image to be built + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built - id: mypy-dev name: Run mypy for dev language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^dev/.*\.py$|^scripts/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1429,7 +1419,7 @@ repos: stages: ['manual'] name: Run mypy for dev (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py dev + entry: ./scripts/ci/prek/mypy_folder.py dev pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1437,7 +1427,7 @@ repos: - id: mypy-airflow-core name: Run mypy for airflow-core language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^airflow-core/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1445,7 +1435,7 @@ repos: stages: ['manual'] name: Run mypy for airflow-core (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py airflow-core + entry: ./scripts/ci/prek/mypy_folder.py airflow-core pass_filenames: false files: ^airflow-core/.*\.py$ require_serial: true @@ -1453,7 +1443,7 @@ repos: - id: mypy-providers name: Run mypy for providers language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^providers/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1461,7 +1451,7 @@ repos: stages: ['manual'] name: Run mypy for providers (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py providers + entry: ./scripts/ci/prek/mypy_folder.py providers pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1469,7 +1459,7 @@ repos: - id: mypy-task-sdk name: Run mypy for task-sdk language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^task-sdk/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1477,7 +1467,7 @@ repos: stages: ['manual'] name: Run mypy for task-sdk (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py task-sdk + entry: ./scripts/ci/prek/mypy_folder.py task-sdk pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1485,7 +1475,7 @@ repos: - id: mypy-devel-common name: Run mypy for devel-common language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^devel-common/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1493,7 +1483,7 @@ repos: stages: ['manual'] name: Run mypy for devel-common (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py devel-common + entry: ./scripts/ci/prek/mypy_folder.py devel-common pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1501,7 +1491,7 @@ repos: - id: mypy-airflow-ctl name: Run mypy for airflow-ctl language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^airflow-ctl/src/airflowctl/.*\.py$|^airflow-ctl/tests/.*\.py$ exclude: .*generated.py require_serial: true @@ -1510,7 +1500,7 @@ repos: stages: ['manual'] name: Run mypy for airflow-ctl (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py airflow-ctl + entry: ./scripts/ci/prek/mypy_folder.py airflow-ctl pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1518,7 +1508,7 @@ repos: - id: generate-openapi-spec name: Generate the FastAPI API spec language: python - entry: ./scripts/ci/pre_commit/generate_openapi_spec.py + entry: ./scripts/ci/prek/generate_openapi_spec.py pass_filenames: false files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* @@ -1526,14 +1516,14 @@ repos: - id: generate-openapi-spec-fab name: Generate the FastAPI API spec for FAB language: python - entry: ./scripts/ci/pre_commit/generate_openapi_spec_fab.py + entry: ./scripts/ci/prek/generate_openapi_spec_fab.py pass_filenames: false files: ^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'openapi-spec-validator>=0.7.1'] - id: generate-openapi-spec-keycloak name: Generate the FastAPI API spec for Keycloak language: python - entry: ./scripts/ci/pre_commit/generate_openapi_spec_keycloak.py + entry: ./scripts/ci/prek/generate_openapi_spec_keycloak.py pass_filenames: false files: ^providers/keycloak/src/airflow/providers/keycloak/auth_manager/.*\.py$ additional_dependencies: [ 'rich>=12.4.4', 'openapi-spec-validator>=0.7.1' ] @@ -1542,12 +1532,12 @@ repos: description: Check i18n files are valid json, have no TODOs, and auto-format them language: python files: ^airflow-core/src/airflow/ui/public/i18n/locales/.*\.json$ - entry: ./scripts/ci/pre_commit/check_i18n_json.py + entry: ./scripts/ci/prek/check_i18n_json.py additional_dependencies: ['rich>=12.4.4'] pass_filenames: false - id: check-provider-yaml-valid name: Validate provider.yaml files - entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py + entry: ./scripts/ci/prek/check_provider_yaml_files.py language: python files: ^providers/.*/provider\.yaml$ exclude: ^providers/.*/.venv/.*$ @@ -1556,14 +1546,14 @@ repos: - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python - entry: ./scripts/ci/pre_commit/check_template_fields.py + entry: ./scripts/ci/prek/check_template_fields.py files: ^(providers/.*/)?airflow-core/.*/(sensors|operators)/.*\.py$ additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: update-migration-references name: Update migration ref doc language: python - entry: ./scripts/ci/pre_commit/migration_reference.py + entry: ./scripts/ci/prek/migration_reference.py pass_filenames: false files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] @@ -1587,13 +1577,13 @@ repos: - id: update-er-diagram name: Update ER diagram language: python - entry: ./scripts/ci/pre_commit/update_er_diagram.py + entry: ./scripts/ci/prek/update_er_diagram.py pass_filenames: false files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] - id: check-default-configuration name: Check the default configuration - entry: ./scripts/ci/pre_commit/check_default_configuration.py + entry: ./scripts/ci/prek/check_default_configuration.py language: python additional_dependencies: ['rich>=12.4.4'] require_serial: true @@ -1601,7 +1591,7 @@ repos: files: ^airflow-core/src/airflow/config_templates/config\.yml$ - id: generate-airflowctl-help-images name: Generate SVG from Airflow CTL Commands - entry: ./scripts/ci/pre_commit/capture_airflowctl_help.py + entry: ./scripts/ci/prek/capture_airflowctl_help.py language: python pass_filenames: false files: @@ -1609,14 +1599,14 @@ repos: additional_dependencies: ['rich>=12.4.4', 'argcomplete>=1.10'] - id: check-imports-in-providers name: Check imports in providers - entry: ./scripts/ci/pre_commit/check_imports_in_providers.py + entry: ./scripts/ci/prek/check_imports_in_providers.py language: python additional_dependencies: ['rich>=12.4.4', 'ruff==0.12.8'] files: ^providers/.*/src/airflow/providers/.*version_compat.*\.py$ require_serial: true - id: provider-version-compat name: Check for correct version_compat imports in providers - entry: ./scripts/ci/pre_commit/check_provider_version_compat.py + entry: ./scripts/ci/prek/check_provider_version_compat.py language: python types: [python] files: ^providers/.*/src/airflow/providers/.*\.py$ @@ -1638,7 +1628,7 @@ repos: ^airflow-core/tests/unit/models/test_variable\.py$ - id: check-sdk-imports name: Check for SDK imports in core files - entry: ./scripts/ci/pre_commit/check_sdk_imports.py + entry: ./scripts/ci/prek/check_sdk_imports.py language: python types: [python] files: ^airflow-core/src/airflow/ @@ -1733,4 +1723,4 @@ repos: ^airflow-core/src/airflow/serialization/helpers\.py$| ^airflow-core/src/airflow/models/expandinput\.py$ additional_dependencies: ['rich>=12.4.4'] - ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE diff --git a/.rat-excludes b/.rat-excludes index 48939c413c9b3..129d7210f07b1 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -179,3 +179,6 @@ www-hash.txt go.mod go.sum mocks/* + +# Kubernetes env +.env diff --git a/AGENTS.md b/AGENTS.md index 724896d268710..43afab4e7db41 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,10 +27,10 @@ how to set up the environment, run checks, build docs and follow the PR workflow - [`07_local_virtualenv.rst`](contributing-docs/07_local_virtualenv.rst) explains how to prepare a local Python environment using `uv`. The tool creates and syncs a `.venv` and installs dependencies with commands such as `uv venv` and `uv sync`. - [`06_development_environments.rst`](contributing-docs/06_development_environments.rst) compares the local virtualenv with the Docker based Breeze environment. Breeze replicates CI and includes services like databases for integration tests. -## Pre-commit hooks +## Prek hooks -- Installation and usage of `pre-commit` are described in [`03_contributors_quick_start.rst`](contributing-docs/03_contributors_quick_start.rst). Install with `uv tool install pre-commit --with pre-commit-uv` and run checks via `pre-commit run --all-files`. -- [`08_static_code_checks.rst`](contributing-docs/08_static_code_checks.rst) provides more details on the available hooks and prerequisites. Enable the hooks with `pre-commit install` so they run automatically on each commit. +- Installation and usage of `prek` are described in [`03_contributors_quick_start.rst`](contributing-docs/03_contributors_quick_start.rst). Install with `uv tool install prek` and run checks via `prek --all-files`. +- [`08_static_code_checks.rst`](contributing-docs/08_static_code_checks.rst) provides more details on the available hooks and prerequisites. Enable the hooks with `prek install` so they run automatically on each commit. ## Running tests diff --git a/Dockerfile b/Dockerfile index de7624482e862..4daadc214c63f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ ARG PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" ARG AIRFLOW_PIP_VERSION=25.2 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0 -ARG AIRFLOW_UV_VERSION=0.8.9 +ARG AIRFLOW_UV_VERSION=0.8.11 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" @@ -87,7 +87,7 @@ FROM scratch as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be -# replaced by pre-commit automatically from the "scripts/docker/" folder. +# replaced by prek automatically from the "scripts/docker/" folder. # This is done in order to avoid problems with caching and file permissions and in order to # make the PROD Dockerfile standalone ############################################################################################## @@ -618,20 +618,19 @@ function common::install_packaging_tools() { pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi fi - if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then + if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then echo - echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}" + echo "${COLOR_BLUE}Installing latest preh, uv${COLOR_RESET}" echo - uv tool install pre-commit --with pre-commit-uv --with uv + uv tool install prek --with uv # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" else echo - echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}" - echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}" + echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}" + echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}" echo - uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \ - --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}" + uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}" # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" fi diff --git a/Dockerfile.ci b/Dockerfile.ci index 967e2ebfebc96..30d65032149ec 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -26,7 +26,7 @@ FROM ${BASE_IMAGE} as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be -# replaced by pre-commit automatically from the "scripts/docker/" folder. +# replaced by prek automatically from the "scripts/docker/" folder. # This is done in order to avoid problems with caching and file permissions and in order to # make the PROD Dockerfile standalone ############################################################################################## @@ -587,20 +587,19 @@ function common::install_packaging_tools() { pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi fi - if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then + if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then echo - echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}" + echo "${COLOR_BLUE}Installing latest preh, uv${COLOR_RESET}" echo - uv tool install pre-commit --with pre-commit-uv --with uv + uv tool install prek --with uv # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" else echo - echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}" - echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}" + echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}" + echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}" echo - uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \ - --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}" + uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}" # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" fi @@ -1334,7 +1333,7 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ARG AIRFLOW_PYTHON_VERSION=v3.10.18 ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION -ENV GOLANG_MAJOR_MINOR_VERSION=1.24.6 +ENV GOLANG_MAJOR_MINOR_VERSION=1.25.0 COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/ @@ -1465,17 +1464,16 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe ARG AIRFLOW_PIP_VERSION=25.2 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0 -ARG AIRFLOW_UV_VERSION=0.8.9 +ARG AIRFLOW_UV_VERSION=0.8.11 # TODO(potiuk): automate with upgrade check (possibly) -ARG AIRFLOW_PRE_COMMIT_VERSION="4.3.0" -ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" +ARG AIRFLOW_PREK_VERSION="0.0.26" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_SETUPTOOLS_VERSION=${AIRFLOW_SETUPTOOLS_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ # This is needed since we are using cache mounted from the host UV_LINK_MODE=copy \ - AIRFLOW_PRE_COMMIT_VERSION=${AIRFLOW_PRE_COMMIT_VERSION} + AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION} # The PATH is needed for PIPX to find the tools installed and cargo to build the wheels ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}" diff --git a/INSTALL b/INSTALL index 59e596ca79350..7d4cc066d983e 100644 --- a/INSTALL +++ b/INSTALL @@ -80,12 +80,12 @@ by running the following command: uv sync --all-packages With `uv` you can also install tools that are needed for other tasks described later - breeze, -pre-commit, hatch, cherry-picker etc. It is highly recommended to install breeze and pre-commit, hatch +prek, hatch, cherry-picker etc. It is highly recommended to install breeze and prek, hatch and flit are useful to build packages (so might be useful by release managers), and cherry-picker is useful for backporting changes to previous versions of Airflow. uv tool install -e ./dev/breeze - uv tool install pre-commit --with pre-commit-uv + uv tool install prek uv tool install hatch uv tool install flit @@ -94,10 +94,10 @@ useful for backporting changes to previous versions of Airflow. Those are all tools useful for Airflow development. -It is recommended to run `pre-commit install` after installing `pre-commit` to install the git hooks - they -will take care about running airflow pre-commit checks automatically. +It is recommended to run `prek install` after installing `prek` to install the git hooks - they +will take care about running airflow prek checks automatically. - pre-commit install + prek install You can run any command in the virtual environment created by `uv` by prefixing it with `uv run`: @@ -112,16 +112,16 @@ Compiling front-end assets In order to see UI in Airflow, you need to compile front-end assets first. -In case you already installed `breeze` and `pre-commit`, you can build the assets with +In case you already installed `breeze` and `prek`, you can build the assets with the following commands: - pre-commit run --hook-stage manual compile-ui-assets --all-files + prek --hook-stage manual compile-ui-assets --all-files or simply: breeze compile-ui-assets -Both commands will install node and pmpm under the hood, to a dedicated pre-commit +Both commands will install node and pmpm under the hood, to a dedicated prek node environment and then build the assets. If you want to manually run the build check for node and pnpm version in `.pre-commit-config.yaml` file, @@ -140,7 +140,7 @@ This will also clean-up the generated files,update `git_version` file in the Air it in the UI) that should contain the git commit hash of the build and it will generate the assets. The result of this command is airflow sdist package built in the `dist` folder of `airflow-core` -package as well. It requires ``pre-commit`` to be installed in your system. +package as well. It requires ``prek`` to be installed in your system. Using pip and manually managing your virtualenv =============================================== diff --git a/PROVIDERS.rst b/PROVIDERS.rst index ef3239ae6ab35..a166181038315 100644 --- a/PROVIDERS.rst +++ b/PROVIDERS.rst @@ -348,5 +348,5 @@ The dependencies for Airflow providers are managed in the ``provider.yaml`` file All provider dependencies, including versions and constraints, are listed in this file. When adding or updating a provider or its dependencies, changes should be made to this file accordingly. -To ensure consistency and manage dependencies, ``pre-commit`` is configured to automatically update all dependencies. -Once you have ``pre-commit`` installed, it will automatically handle the dependency updates. +To ensure consistency and manage dependencies, ``prek`` is configured to automatically update all dependencies. +Once you have ``prek`` installed, it will automatically handle the dependency updates. diff --git a/README.md b/README.md index eb911e71ecf05..52e432c57f858 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ packages: Apache Airflow version life cycle: - + | Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index c418a65ebc6c0..1fe94d1381f19 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1491,7 +1491,7 @@ Miscellaneous - Bump ``dompurify`` from 2.2.9 to 2.5.6 in /airflow/www (#42263) (#42270) - Correct docstring format in _get_template_context (#42244) (#42272) - Backport: Bump Flask-AppBuilder to ``4.5.2`` (#43309) (#43318) -- Check python version that was used to install pre-commit venvs (#43282) (#43310) +- Check python version that was used to install prek venvs (#43282) (#43310) - Resolve warning in Dataset Alias migration (#43425) Doc Only Changes diff --git a/airflow-core/docs/database-erd-ref.rst b/airflow-core/docs/database-erd-ref.rst index 80673a336ab1b..3f9ea2c77fc65 100644 --- a/airflow-core/docs/database-erd-ref.rst +++ b/airflow-core/docs/database-erd-ref.rst @@ -33,6 +33,6 @@ Here is the current Database schema diagram. `db command `_ for the commands that you can use to manage the migrations. -.. This image is automatically generated by pre-commit via ``scripts/ci/pre_commit/update_er_diagram.py`` +.. This image is automatically generated by prek hook via ``scripts/ci/prek/update_er_diagram.py`` .. image:: img/airflow_erd.svg diff --git a/airflow-core/docs/howto/custom-operator.rst b/airflow-core/docs/howto/custom-operator.rst index d6206166e1211..ad09760c25444 100644 --- a/airflow-core/docs/howto/custom-operator.rst +++ b/airflow-core/docs/howto/custom-operator.rst @@ -349,7 +349,7 @@ Thus, the following example is valid: class MyHelloOperator(HelloOperator): template_fields = "foo" -The limitations above are enforced by a pre-commit named 'validate-operators-init'. +The limitations above are enforced by a prek hook named 'validate-operators-init'. Add template fields with subclassing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/docs/howto/usage-cli.rst b/airflow-core/docs/howto/usage-cli.rst index a7dcc71abba4d..ce9869b0d419b 100644 --- a/airflow-core/docs/howto/usage-cli.rst +++ b/airflow-core/docs/howto/usage-cli.rst @@ -384,7 +384,7 @@ JSON example output: Testing for DAG Import Errors ----------------------------- The CLI can be used to check whether any discovered dags have import errors via the ``list-import-errors`` subcommand. It is possible to create an automation step which fails if any dags cannot be imported by checking the command output, particularly when used with ``--output`` to generate a standard file format. -For example, the default output when there are no errors is ``No data found``, and the json output is ``[]``. The check can then be run in CI or pre-commit to speed up the review process and testing. +For example, the default output when there are no errors is ``No data found``, and the json output is ``[]``. The check can then be run in CI or as prek hook to speed up the review process and testing. Example command that fails if there are any errors, using `jq `__ to parse the output: diff --git a/airflow-core/docs/installation/supported-versions.rst b/airflow-core/docs/installation/supported-versions.rst index a8ba1db336805..9ff6ed02aa720 100644 --- a/airflow-core/docs/installation/supported-versions.rst +++ b/airflow-core/docs/installation/supported-versions.rst @@ -23,7 +23,7 @@ Version Life Cycle Apache Airflow® version life cycle: - .. This table is automatically updated by pre-commit scripts/ci/pre_commit/supported_versions.py + .. This table is automatically updated by prek hook scripts/ci/prek/supported_versions.py .. Beginning of auto-generated table ========= ===================== ========= =============== ===================== ================ diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index 6d952e0c808d3..759db30276e65 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -32,7 +32,7 @@ Here's the list of all the Database Migrations that are executed via when you ru - .. This table is automatically updated by pre-commit by ``scripts/ci/pre_commit/migration_reference.py`` + .. This table is automatically updated by prek hook by ``scripts/ci/prek/migration_reference.py`` .. All table elements are scraped from migration files .. Beginning of auto-generated table diff --git a/airflow-core/hatch_build.py b/airflow-core/hatch_build.py index d691e6c25cd1d..018067a87a9ae 100644 --- a/airflow-core/hatch_build.py +++ b/airflow-core/hatch_build.py @@ -65,7 +65,7 @@ def build_standard(self, directory: str, artifacts: Any, **build_data: Any) -> s self.write_git_version() # run this in the parent directory of the airflow-core (i.e. airflow repo root) work_dir = Path(self.root).parent.resolve() - cmd = ["pre-commit", "run", "--hook-stage", "manual", "compile-ui-assets", "--all-files"] + cmd = ["prek", "run", "--hook-stage", "manual", "compile-ui-assets", "--all-files"] log.warning("Running command: %s", " ".join(cmd)) run(cmd, cwd=work_dir.as_posix(), check=True) dist_path = Path(self.root) / "src" / "airflow" / "ui" / "dist" diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index a73382c2c720e..f5a2eb72acacc 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -62,7 +62,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] -# Version is defined in src/airflow/__init__.py and it is automatically synchronized by pre-commit +# Version is defined in src/airflow/__init__.py and it is automatically synchronized by prek hook version = "3.1.0" dependencies = [ diff --git a/airflow-core/src/airflow/ui/public/i18n/README.md b/airflow-core/src/airflow/ui/public/i18n/README.md index 4fa50122cf7a4..536eb3013d0e1 100644 --- a/airflow-core/src/airflow/ui/public/i18n/README.md +++ b/airflow-core/src/airflow/ui/public/i18n/README.md @@ -258,7 +258,7 @@ uv run dev/i18n/check_translations_completeness.py --language -- > The following describe the desired future state of compliance and enforcement. - Automated checks SHOULD verify once in a while that all languages have corresponding entries for new terms in the default language. When translations are missing, relevant code owners should be notified. -- Automated checks SHOULD allow a person doing translation to select the language and aid them in adding new translations so that they do not have to compare them manually. Possibly it can be done by adding `-–add-missing` to the verifying script that will add new entries with `TODO: translate: ENGLISH VERSION` and add pre-commit to not allow such `TODO:` entries to be committed. +- Automated checks SHOULD allow a person doing translation to select the language and aid them in adding new translations so that they do not have to compare them manually. Possibly it can be done by adding `-–add-missing` to the verifying script that will add new entries with `TODO: translate: ENGLISH VERSION` and add prek hook to not allow such `TODO:` entries to be committed. ## 10. Exceptions diff --git a/airflow-ctl/docs/installation/supported-versions.rst b/airflow-ctl/docs/installation/supported-versions.rst index 823d4e5b46149..755426c7b6ba1 100644 --- a/airflow-ctl/docs/installation/supported-versions.rst +++ b/airflow-ctl/docs/installation/supported-versions.rst @@ -25,7 +25,7 @@ Apache Airflow CTL is compatible with Apache Airflow® versions 3.0.0 and later. Apache Airflow CTL version life cycle: - .. This table is automatically updated by pre-commit scripts/ci/pre_commit/supported_versions.py + .. This table is automatically updated by prek hook scripts/ci/prek/supported_versions.py .. Beginning of auto-generated table ========= ===================== ========= =============== ================= ================ diff --git a/contributing-docs/02_how_to_communicate.rst b/contributing-docs/02_how_to_communicate.rst index e62341e65ec65..1d0739c2c4095 100644 --- a/contributing-docs/02_how_to_communicate.rst +++ b/contributing-docs/02_how_to_communicate.rst @@ -121,7 +121,7 @@ and loosely organized team. We have both - contributors that commit one commit o more commits. It happens that some people assume informal "stewardship" over parts of code for some time - but at any time we should make sure that the code can be taken over by others, without excessive communication. Setting high requirements for the code (fairly strict code review, static code checks, requirements of -automated tests, pre-commit checks) is the best way to achieve that - by only accepting good quality +automated tests, prek hooks) is the best way to achieve that - by only accepting good quality code. Thanks to full test coverage we can make sure that we will be able to work with the code in the future. So do not be surprised if you are asked to add more tests or make the code cleaner - this is for the sake of maintainability. diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index fb556ae9061fa..d36e777b87a6f 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -234,8 +234,8 @@ Forking and cloning Project git config core.autocrlf true -Configuring Pre-commit ----------------------- +Configuring prek +---------------- Before committing changes to github or raising a pull request, the code needs to be checked for certain quality standards such as spell check, code syntax, code formatting, compatibility with Apache License requirements etc. This set of @@ -249,7 +249,7 @@ tests are applied when you commit your code. -To avoid burden on our CI infrastructure and to save time, Pre-commit hooks can be run locally before committing changes. +To avoid burden on our CI infrastructure and to save time, prek hooks can be run locally before committing changes. .. note:: We have recently started to recommend ``uv`` for our local development. @@ -258,7 +258,7 @@ To avoid burden on our CI infrastructure and to save time, Pre-commit hooks can Remember to have global python set to Python >= 3.10 - Python 3.10 is end-of-life already and we've started to use Python 3.10+ features in Airflow and accompanying scripts. -Installing pre-commit is best done with ``uv`` (recommended) or ``pipx``. +Installing prek is best done with ``uv`` (recommended) or ``pipx``. 1. Installing required packages @@ -274,7 +274,7 @@ on macOS, install via brew install libxml2 -2. Installing pre-commit: +2. Installing prek: .. note:: You might need to pass ``--python `` to force the python version if not it uses the latest system python version. @@ -282,15 +282,14 @@ on macOS, install via .. code-block:: bash - uv tool install pre-commit --with pre-commit-uv + uv tool install prek -You can add ``uv`` support for ``pre-commit`` even if you've installed it with ``pipx`` using the commands -(then pre-commit will use ``uv`` to create virtualenvs for the hooks): +or with pipx: .. code-block:: bash - pipx install pre-commit - pipx install inject pre-commit pre-commit-uv # optional, configures pre-commit to use uv to install virtualenvs + pipx install prek + 3. Go to your project directory @@ -299,11 +298,11 @@ You can add ``uv`` support for ``pre-commit`` even if you've installed it with ` cd ~/Projects/airflow -4. Running pre-commit hooks +4. Running prek hooks .. code-block:: bash - pre-commit run --all-files + prek --all-files No-tabs checker......................................................Passed Add license for all SQL files........................................Passed Add license for all other files......................................Passed @@ -327,39 +326,39 @@ You can add ``uv`` support for ``pre-commit`` even if you've installed it with ` Fix End of Files.....................................................Passed ........................................................................... -5. Running pre-commit for selected files +5. Running prek for selected files .. code-block:: bash - pre-commit run --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py + prek --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py 6. Running specific hook for selected files .. code-block:: bash - pre-commit run black --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py + prek black --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py black...............................................................Passed - pre-commit run ruff --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py + prek ruff --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py Run ruff............................................................Passed -7. Enabling Pre-commit check before push +7. Enabling prek hook check before push -It will run pre-commit automatically before committing and stops the commit on failure +It will run prek hooks automatically before committing and stops the commit on failure .. code-block:: bash cd ~/Projects/airflow - pre-commit install + prek install git commit -m "Added xyz" -8. To disable Pre-commit +8. To disable prek hooks .. code-block:: bash cd ~/Projects/airflow - pre-commit uninstall + prek uninstall - For more information on this visit |08_static_code_checks.rst| @@ -370,12 +369,12 @@ It will run pre-commit automatically before committing and stops the commit on f - Following are some of the important links of 08_static_code_checks.rst - - |Pre-commit Hooks| + - |Prek Hooks| - .. |Pre-commit Hooks| raw:: html + .. |Prek Hooks| raw:: html - - Pre-commit Hooks + + Prek Hooks - |Running Static Code Checks via Breeze| diff --git a/contributing-docs/05_pull_requests.rst b/contributing-docs/05_pull_requests.rst index 5b7c5f93f9510..2bad72563f360 100644 --- a/contributing-docs/05_pull_requests.rst +++ b/contributing-docs/05_pull_requests.rst @@ -36,13 +36,13 @@ these guidelines: run the tests and `codecov `__ to track coverage. You can set up both for free on your fork. It will help you make sure you do not break the build with your PR and that you help increase coverage. - Also we advise to install locally `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`__ to + Also we advise to install locally `prek hooks <08_static_code_checks.rst#prek-hooks>`__ to apply various checks, code generation and formatting at the time you make a local commit - which gives you near-immediate feedback on things you need to fix before you push your code to the PR, or in many case it will even fix it for you locally so that you can add and commit it straight away. - Follow our project's `Coding style and best practices`_. Usually we attempt to enforce the practices by - having appropriate pre-commits. There are checks amongst them that aren't currently enforced + having appropriate prek hooks. There are checks amongst them that aren't currently enforced programmatically (either because they are too hard or just not yet done). - Maintainers will not merge a PR that regresses linting or does not pass CI tests (unless you have good @@ -62,16 +62,16 @@ these guidelines: maintenance burden during rebase. - Add an `Apache License `__ header to all new files. If you - have ``pre-commit`` installed, pre-commit will do it automatically for you. If you hesitate to install - pre-commit for your local repository - for example because it takes a few seconds to commit your changes, - this one thing might be a good reason to convince anyone to install pre-commit. + have ``prek`` installed, prej will do it automatically for you. If you hesitate to install + prek for your local repository - for example because it takes a few seconds to commit your changes, + this one thing might be a good reason to convince anyone to install prek. - If your PR adds functionality, make sure to update the docs as part of the same PR, not only code and tests. Docstring is often sufficient. Make sure to follow the Sphinx compatible standards. - Make sure your code fulfills all the `static code checks <08_static_code_checks.rst#static-code-checks>`__ we have in our code. The easiest way - to make sure of that is - again - to install `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`__ + to make sure of that is - again - to install `prek hooks <08_static_code_checks.rst#prek-hooks>`__ - Make sure your PR is small and focused on one change only - avoid adding unrelated changes, mixing adding features and refactoring. Keeping to that rule will make it easier to review your PR and will make @@ -271,7 +271,7 @@ In such cases we can usually do something like this self.my_field = my_field The reason for doing it is that we are working on a cleaning up our code to have -`pre-commit hook <../scripts/ci/pre_commit/validate_operators_init.py>`__ +`prek hook <../scripts/ci/prek/validate_operators_init.py>`__ that will make sure all the cases where logic (such as validation and complex conversion) is not done in the constructor are detected in PRs. diff --git a/contributing-docs/06_development_environments.rst b/contributing-docs/06_development_environments.rst index 5ce138a3f1c83..2c5db202b25a1 100644 --- a/contributing-docs/06_development_environments.rst +++ b/contributing-docs/06_development_environments.rst @@ -57,9 +57,9 @@ Limitations: Breeze container-based solution provides a reproducible environment that is consistent with other developers. -- You are **STRONGLY** encouraged to also install and use `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`_ +- You are **STRONGLY** encouraged to also install and use `prek hooks <08_static_code_checks.rst#prekt-hooks>`_ for your local virtualenv development environment. - Pre-commit hooks can speed up your development cycle a lot. + Prek hooks can speed up your development cycle a lot. Typically you can connect your local virtualenv environments easily with your IDE and use it for development: diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index 35ddb4b6f1a1d..82e823140a6ba 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -70,7 +70,7 @@ The ``uv`` utility is a build frontend tool that is designed to manage python, v and testing of Python projects. It is a modern tool that is designed to work with PEP 517/518 compliant projects and it is much faster than "reference" ``pip`` tool. It has extensive support to not only create development environment but also to manage python versions, development environments, workspaces and Python tools used -to develop Airflow (via ``uv tool`` command - such as ``pre-commit`` and others, you can also use ``uv tool`` +to develop Airflow (via ``uv tool`` command - such as ``prek`` and others, you can also use ``uv tool`` to install ``breeze`` - containerized development environment for Airflow that we use to reproduce the CI environment locally and to run release-management and certain development tasks. @@ -247,7 +247,7 @@ dependencies, and sometimes ``development dependencies`` (in ``dev`` dependency to run tests and are installed automatically when you install environment with ``uv-sync``. If you want to add another dependency to a provider, you should add it to corresponding ``pyproject.toml``, -add the files to your commit with ``git add`` and run ``pre-commit run`` to update generated dependencies. +add the files to your commit with ``git add`` and run ``prek`` to update generated dependencies. Note that in the future we will remove that step. For ``uv`` it's simple, you need to run ``uv sync`` in providers directory after you modified diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 1436260fd2ee1..5e910a31a6bfe 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -19,472 +19,195 @@ Static code checks ================== The static code checks in Airflow are used to verify that the code meets certain quality standards. -All the static code checks can be run through pre-commit hooks. +All the static code checks can be run through prek hooks. -The pre-commit hooks perform all the necessary installation when you run them -for the first time. See the table below to identify which pre-commit checks require the Breeze Docker images. +The prek hooks perform all the necessary installation when you run them +for the first time. See the table below to identify which prek checks require the Breeze Docker images. You can also run the checks via `Breeze <../dev/breeze/doc/README.rst>`_ environment. **The outline for this document in GitHub is available at top-right corner button (with 3-dots and 3 lines).** -Pre-commit hooks ----------------- +Prek hooks +---------- Pre-commit hooks help speed up your local development cycle and place less burden on the CI infrastructure. -Consider installing the pre-commit hooks as a necessary prerequisite. +Consider installing the prek hooks as a necessary prerequisite. -The pre-commit hooks by default only check the files you are currently working on and make -them fast. Yet, these checks use exactly the same environment as the CI tests +The hooks by default only check the files you are currently working on (and are staged) which makes the +checks rather fast. Yet, these checks use exactly the same environment as the CI tests use. So, you can be sure your modifications will also work for CI if they pass -pre-commit hooks. +prek hooks. -We have integrated the fantastic `pre-commit `__ framework -in our development workflow. To install and use it, you need at least Python 3.10 locally. +We have integrated the `prek `__ framework +in our development workflow. It can be installed in various ways and does not even need ``pip`` or -Installing pre-commit hooks ---------------------------- +``python`` to be installed. It is a drop-in replacement for the legacy ``pre-commit`` tool, but it is +much faster and more feature-rich. It is written in Rust and it is designed to install environments in parallel, +so it is much faster than the ``pre-commit`` tool. + +Installing prek hooks +--------------------- + +It is the best to use prek hooks when you have your local virtualenv for +Airflow activated since then prek hooks and other dependencies are +automatically installed. You can also install the prek hooks manually using ``uv`` or ``pipx``. -It is the best to use pre-commit hooks when you have your local virtualenv for -Airflow activated since then pre-commit hooks and other dependencies are -automatically installed. You can also install the pre-commit hooks manually using ``uv`` or ``pip``. +.. code-block:: bash + + uv tool install prek .. code-block:: bash - uv tool install pre-commit + pipx install prek + +Since we have a lot of hooks and sometimes you want to run them individually, it's advised to install +auto-completion for the ``prek`` command. You can do it by adding the following line to your +``.bashrc`` or ``.zshrc`` file: + +For bash: .. code-block:: bash - pip install pre-commit + eval "$(COMPLETE=bash prek)" # for bash + +For zsh: + +.. code-block:: zsh -After installation, pre-commit hooks are run automatically when you commit the code and they will + eval "$(COMPLETE=zsh prek)" + +Similarly for other shells like fish, powershell, etc. + +After installation, prek hooks are run automatically when you commit the code and they will only run on the files that you change during your commit, so they are usually pretty fast and do -not slow down your iteration speed on your changes. There are also ways to disable the ``pre-commits`` +not slow down your iteration speed on your changes. There are also ways to disable the prek hooks temporarily when you commit your code with ``--no-verify`` switch or skip certain checks that you find -to much disturbing your local workflow. See `Available pre-commit checks <#available-pre-commit-checks>`_ -and `Using pre-commit <#using-pre-commit>`_ +to much disturbing your local workflow. See `Using prek <#using-prek>`_ -The pre-commit hooks use several external linters that need to be installed before pre-commit is run. +The ``prek`` hooks use several external linters that need to be installed before prek is run. Each of the checks installs its own environment, so you do not need to install those, but there are some checks that require locally installed binaries. On Linux, you typically install them with ``sudo apt install``, on macOS - with ``brew install``. -The current list of prerequisites is limited to ``xmllint``: +The current list of prerequisites is limited to ``xmllint`` and ``golang`` if you want to modify +the Golang code.: -- on Linux, install via ``sudo apt install libxml2-utils`` -- on macOS, install via ``brew install libxml2`` +- on Linux, install via ``sudo apt install libxml2-utils golang`` +- on macOS, install via ``brew install libxml2 golang`` -Some pre-commit hooks also require the Docker Engine to be configured as the static -checks are executed in the Docker environment (See table in the -`Available pre-commit checks <#available-pre-commit-checks>`_ . You should build the images -locally before installing pre-commit checks as described in `Breeze docs <../dev/breeze/doc/README.rst>`__. +Some prek hooks also require the Docker Engine to be configured as the static +checks are executed in the Docker environment. You should build the images +locally before installing prek checks as described in `Breeze docs <../dev/breeze/doc/README.rst>`__. Sometimes your image is outdated and needs to be rebuilt because some dependencies have been changed. -In such cases, the Docker-based pre-commit will inform you that you should rebuild the image. +In such cases, the Docker-based prek will inform you that you should rebuild the image. -In case you do not have your local images built, the pre-commit hooks fail and provide -instructions on what needs to be done. +Enabling prek hooks +------------------- -Enabling pre-commit hooks -------------------------- +To turn on prek checks for ``commit`` operations in git, enter: -To turn on pre-commit checks for ``commit`` operations in git, enter: +.. code-block:: bash + + prek install + +To install the checks also for ``pre-push`` operations, enter: .. code-block:: bash - pre-commit install + prek install -t pre-push +For details on advanced usage of the install method, use: -To install the checks also for ``pre-push`` operations, enter: +.. code-block:: bash + + prek install --help + +.. note:: + + The ``prek`` tool is a drop-in replacement for the legacy ``pre-commit`` tool - much faster and more + feature-rich, If you have already installed ``pre-commit`` to handle your hooks, you can run + ``prek install -f`` to replace the existing ``pre-commit`` hooks with the ``prek`` hooks. + +Available prek hooks +-------------------- + +You can see the list of available hooks by running: .. code-block:: bash - pre-commit install -t pre-push + prek list +You can also see more details about the hooks by running: -For details on advanced usage of the install method, use: +.. code-block:: bash + + prek list --verbose + +And if you want to see the details of a particular hook, you can run: .. code-block:: bash - pre-commit install --help + prek list --verbose -Available pre-commit checks ---------------------------- +When you install auto-completion, you can also use the tab-completion to see the available hooks. -This table lists pre-commit hooks used by Airflow. The ``Image`` column indicates which hooks -require Breeze Docker image to be built locally. - - .. BEGIN AUTO-GENERATED STATIC CHECK LIST - -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ID | Description | Image | -+===========================================================+========================================================+=========+ -| bandit | bandit | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| blacken-docs | Run black on docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-aiobotocore-optional | Check if aiobotocore is an optional dependency only | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-k8s-not-used | Check airflow.kubernetes imports are not used | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-providers-bug-report-template | Sort airflow-bug-report provider list | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-v-imports-in-tests | Check AIRFLOW_V imports in tests | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-version-checks-in-core | No AIRFLOW_V_* imports in airflow-core | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-apache-license-rat | Check if licenses are OK for Apache | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-base-operator-partial-arguments | Check BaseOperator and partial() arguments | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-base-operator-usage | * Check BaseOperator core imports | | -| | * Check BaseOperatorLink core imports | | -| | * Check BaseOperator other imports | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-boring-cyborg-configuration | Checks for Boring Cyborg configuration consistency | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-breeze-top-dependencies-limited | Check top-level breeze deps | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-builtin-literals | Require literal syntax when initializing builtins | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-changelog-format | Check changelog format | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-changelog-has-no-duplicates | Check changelogs for duplicate entries | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-cncf-k8s-only-for-executors | Check cncf.kubernetes imports used for executors only | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-code-deprecations | Check deprecations categories in decorators | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-common-compat-used-for-openlineage | Check common.compat is used for OL deprecated classes | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-core-deprecation-classes | Verify usage of Airflow deprecation classes in core | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-decorated-operator-implements-custom-name | Check @task decorator implements custom_operator_name | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-default-configuration | Check the default configuration | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-deferrable-default | Check and fix default value of default_deferrable | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-docstring-param-types | Check that docstrings do not specify param types | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-example-dags-urls | Check that example dags url include provider versions | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-executables-have-shebangs | Check that executables have shebang | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-extra-packages-references | Checks setup extra packages | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-extras-order | Check order of extras in Dockerfile | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-fab-migrations | Check no migration is done on FAB related table | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-for-inclusive-language | Check for language that we do not accept as community | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-get-lineage-collector-providers | Check providers import hook lineage code from compat | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-hooks-apply | Check if all hooks apply to the repository | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-i18n-json | Check i18n files validity | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-imports-in-providers | Check imports in providers | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-incorrect-use-of-LoggingMixin | Make sure LoggingMixin is not used alone | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-init-decorator-arguments | Sync model __init__ and decorator arguments | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-integrations-list-consistent | Sync integrations list with docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-lazy-logging | Check that all logging methods are lazy | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-links-to-example-dags-do-not-use-hardcoded-versions | Verify no hard-coded version in example dags | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-merge-conflict | Check that merge conflicts are not being committed | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-min-python-version | Check minimum Python version | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-newsfragments-are-valid | Check newsfragments are valid | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-no-airflow-deprecation-in-providers | Do not use DeprecationWarning in providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-no-providers-in-core-examples | No providers imports in core example DAGs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-only-new-session-with-provide-session | Check NEW_SESSION is only used with @provide_session | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-persist-credentials-disabled-in-github-workflows | Check persistent creds in workflow files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-pre-commit-information-consistent | Validate hook IDs & names and sync with docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provide-create-sessions-imports | Check session util imports | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provider-docs-valid | Validate provider doc files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provider-yaml-valid | Validate provider.yaml files | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-providers-subpackages-init-file-exist | Provider subpackage init files are there | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-pydevd-left-in-code | Check for pydevd debug statements accidentally left | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-pytest-mark-db-test-in-providers | Check pytest.mark.db_test use in providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-revision-heads-map | Check that the REVISION_HEADS_MAP is up-to-date | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-safe-filter-usage-in-html | Don't use safe in templates | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-sdk-imports | Check for SDK imports in core files | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-shared-distributions-structure | Check shared distributions structure | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-shared-distributions-usage | Check shared distributions usage | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-significant-newsfragments-are-valid | Check significant newsfragments are valid | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-sql-dependency-common-data-structure | Check dependency of SQL providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-start-date-not-used-in-defaults | start_date not in default_args | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-system-tests-present | Check if system tests have required segments of code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-system-tests-tocs | Check that system tests is properly added | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-taskinstance-tis-attrs | Check that TI and TIS have the same attributes | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-template-context-variable-in-sync | Sync template context variable refs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-template-fields-valid | Check templated fields mapped in operators/sensors | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-tests-in-the-right-folders | Check if tests are in the right folders | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-tests-unittest-testcase | Unit tests do not inherit from unittest.TestCase | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-urlparse-usage-in-code | Don't use urlparse in code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-xml | Check XML files with xmllint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-zip-file-is-not-committed | Check no zip files are committed | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| codespell | Run codespell | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-fab-assets | Compile FAB provider assets | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-ui-assets | Compile ui assets (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-ui-assets-dev | Compile ui assets in dev mode (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| create-missing-init-py-files-tests | Create missing init.py files in tests | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| debug-statements | Detect accidentally committed debug statements | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| detect-private-key | Detect if private key is added to the repository | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| doctoc | Add TOC for Markdown and RST files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| end-of-file-fixer | Make sure that there is an empty line at the end | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| flynt | Run flynt string format converter for Python | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| gci | Consistent import ordering for Go files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-airflow-diagrams | Generate airflow diagrams | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-airflowctl-datamodels | Generate Datamodels for AirflowCTL | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-airflowctl-help-images | Generate SVG from Airflow CTL Commands | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-openapi-spec | Generate the FastAPI API spec | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-openapi-spec-fab | Generate the FastAPI API spec for FAB | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-openapi-spec-keycloak | Generate the FastAPI API spec for Keycloak | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-pypi-readme | Generate PyPI README | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-tasksdk-datamodels | Generate Datamodels for TaskSDK client | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-volumes-for-sources | Generate volumes for docker compose | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| go-mockery | Generate mocks for go | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| go-mod-tidy | Run go mod tidy | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| gofmt | Format go code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| identity | Print checked files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| insert-license | * Add license for all SQL files | | -| | * Add license for all RST files | | -| | * Add license for CSS/JS/JSX/PUML/TS/TSX | | -| | * Add license for all Shell files | | -| | * Add license for all toml files | | -| | * Add license for all Python files | | -| | * Add license for all XML files | | -| | * Add license for all Helm template files | | -| | * Add license for all YAML files except Helm templates | | -| | * Add license for all Markdown files | | -| | * Add license for all other files | | -| | * Add license for all Go files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| kubeconform | Kubeconform check on our helm chart | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-chart-schema | Lint chart/values.schema.json file | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-dockerfile | Lint Dockerfile | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-helm-chart | Lint Helm Chart | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-json-schema | * Lint JSON Schema files | | -| | * Lint NodePort Service | | -| | * Lint Docker compose files | | -| | * Lint chart/values.schema.json | | -| | * Lint chart/values.yaml | | -| | * Lint config_templates/config.yml | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-markdown | Run markdownlint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mixed-line-ending | Detect if mixed line ending is used (\r vs. \r\n) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-airflow-core | * Run mypy for airflow-core | * | -| | * Run mypy for airflow-core (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-airflow-ctl | * Run mypy for airflow-ctl | * | -| | * Run mypy for airflow-ctl (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-dev | * Run mypy for dev | * | -| | * Run mypy for dev (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-devel-common | * Run mypy for devel-common | * | -| | * Run mypy for devel-common (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-providers | * Run mypy for providers | * | -| | * Run mypy for providers (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-task-sdk | * Run mypy for task-sdk | * | -| | * Run mypy for task-sdk (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| pretty-format-json | Format JSON files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| prevent-deprecated-sqlalchemy-usage | Prevent deprecated sqlalchemy usage | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| provider-version-compat | Check for correct version_compat imports in providers | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| pylint | pylint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| python-no-log-warn | Check if there are no deprecate log warn | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| replace-bad-characters | Replace bad characters | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| rst-backticks | Check if RST files use double backticks for code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ruff | Run 'ruff' for extremely fast Python linting | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ruff-format | Run 'ruff format' | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| shellcheck | Check Shell scripts syntax correctness | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| trailing-whitespace | Remove trailing whitespace at end of line | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-lint-simple-auth-manager-ui | Compile / format / lint simple auth manager UI | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-lint-ui | Compile / format / lint UI | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-black-version | Update black versions everywhere (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-breeze-cmd-output | Update breeze docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-breeze-readme-config-hash | Update Breeze README.md with config files hash | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-chart-dependencies | Update chart dependencies to latest (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-er-diagram | Update ER diagram | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-in-the-wild-to-be-sorted | Sort INTHEWILD.md alphabetically | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-inlined-dockerfile-scripts | Inline Dockerfile and Dockerfile.ci scripts | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installed-providers-to-be-sorted | Sort and uniquify installed_providers.txt | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installers-and-pre-commit | Update installers and pre-commit to latest (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-local-yml-file | Update mounts in the local yml file | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-migration-references | Update migration ref doc | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-providers-build-files | Update providers build files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-providers-dependencies | Update dependencies for providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-pyproject-toml | Update Airflow's meta-package pyproject.toml | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-reproducible-source-date-epoch | Update Source Date Epoch for reproducible builds | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-spelling-wordlist-to-be-sorted | Sort spelling_wordlist.txt | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-supported-versions | Updates supported versions in documentation | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-vendored-in-k8s-json-schema | Vendor k8s definitions into values.schema.json | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-version | Update versions in docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| validate-chart-annotations | Validate chart annotations | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| validate-operators-init | No templated field logic checks in operator __init__ | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| yamllint | Check YAML files with yamllint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| zizmor | Run zizmor to check for github workflow syntax errors | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ - - .. END AUTO-GENERATED STATIC CHECK LIST - -Using pre-commit ----------------- - -After installation, pre-commit hooks are run automatically when you commit the -code. But you can run pre-commit hooks manually as needed. +Using prek +---------- + +After installation, prek hooks are run automatically when you commit the +code. But you can run prek hooks manually as needed. - Run all checks on your staged files by using: .. code-block:: bash - pre-commit run + prek -- Run only mypy check on your staged files (in ``airflow/`` excluding providers) by using: +- Run only mypy check on your staged airflow and dev files by specifying the + ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified): .. code-block:: bash - pre-commit run mypy-airflow + prek mypy-airflow-core mypy-dev -- Run only mypy checks on all files by using: +- Run only mypy airflow checks on all "airflow-core" files by using: .. code-block:: bash - pre-commit run mypy-airflow --all-files - + prek mypy-airflow-core --all-files - Run all checks on all files by using: .. code-block:: bash - pre-commit run --all-files - + prek --all-files - Run all checks only on files modified in the last locally available commit in your checked out branch: .. code-block:: bash - pre-commit run --source=HEAD^ --origin=HEAD + prek --last-commit + +- Run all checks only on files modified in your last branch that is targeted to be merged into the main branch: + +.. code-block:: bash + prek --from-ref main -- Show files modified automatically by pre-commit when pre-commits automatically fix errors +- Show files modified automatically by prek when prek automatically fix errors .. code-block:: bash - pre-commit run --show-diff-on-failure + prek --show-diff-on-failure - Skip one or more of the checks by specifying a comma-separated list of checks to skip in the SKIP variable: .. code-block:: bash - SKIP=mypy-airflow-core,ruff pre-commit run --all-files + SKIP=mypy-airflow-core,ruff prek --all-files You can always skip running the tests by providing ``--no-verify`` flag to the @@ -495,15 +218,15 @@ To check other usage types of the pre-commit framework, see `Pre-commit website Disabling particular checks --------------------------- -In case you have a problem with running particular ``pre-commit`` check you can still continue using the -benefits of having ``pre-commit`` installed, with some of the checks disabled. In order to disable +In case you have a problem with running particular ``prek`` check you can still continue using the +benefits of having ``prek`` installed, with some of the checks disabled. In order to disable checks you might need to set ``SKIP`` environment variable to coma-separated list of checks to skip. For example, when you want to skip some checks (ruff/mypy for example), you should be able to do it by setting ``export SKIP=ruff,mypy-airflow-core,``. You can also add this to your ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time you enter the terminal. In case you do not have breeze image configured locally, you can also disable all checks that require breeze -the image by setting ``SKIP_BREEZE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically +the image by setting ``SKIP_BREEZE_PREK_HOOKS`` to "true". This will mark the tests as "green" automatically when run locally (note that those checks will anyway run in CI). Disabling goproxy for firewall issues @@ -536,22 +259,22 @@ for more details on how to configure Go proxy - including setting multiple proxi You can add the variable to your ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time you enter the terminal. -Manual pre-commits ------------------- +Manual prek hooks +----------------- Most of the checks we run are configured to run automatically when you commit the code. However, there are some checks that are not run automatically and you need to run them manually. Those checks are marked with ``manual`` in the ``Description`` column in the table below. You can run -them manually by running ``pre-commit run --hook-stage manual ``. +them manually by running ``prek --hook-stage manual ``. -Special pin-versions pre-commit -------------------------------- +Special pin-versions prek +------------------------- -There is a separate pre-commit ``pin-versions`` pre-commit which is used to pin versions of +There is a separate prek ``pin-versions`` prek hook which is used to pin versions of GitHub Actions in the CI workflows. This action requires ``GITHUB_TOKEN`` to be set, otherwise you might hit the rate limits with GitHub API, it -is also configured in a separate ``.pre-commit-config.yaml`` file in the +is also configured in a separate ``.prek-config.yaml`` file in the ``.github`` directory as it requires Python 3.11 to run. It is not run automatically when you commit the code but in runs as a separate job in the CI. However, you can run it manually by running: @@ -559,7 +282,7 @@ manually by running: .. code-block:: bash export GITHUB_TOKEN=YOUR_GITHUB_TOKEN - pre-commit run -c .github/.pre-commit-config.yaml --all-files --hook-stage manual --verbose + prek -c .github/.pre-commit-config.yaml --all-files --hook-stage manual --verbose Mypy checks @@ -579,19 +302,19 @@ command (example for ``airflow`` files): .. code-block:: bash - pre-commit run --hook-stage manual mypy- --all-files + prek --hook-stage manual mypy- --all-files For example: .. code-block:: bash - pre-commit run --hook-stage manual mypy-airflow --all-files + prek --hook-stage manual mypy-airflow --all-files To show unused mypy ignores for any providers/airflow etc, eg: run below command: .. code-block:: bash export SHOW_UNUSED_MYPY_WARNINGS=true - pre-commit run --hook-stage manual mypy-airflow --all-files + prek --hook-stage manual mypy-airflow --all-files MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy @@ -599,99 +322,6 @@ will handle refreshing the cache when and if needed, there are some cases when i is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``. -Running static code checks via Breeze -------------------------------------- - -The static code checks can be launched using the Breeze environment. - -You run the static code checks via ``breeze static-check`` or commands. - -You can see the list of available static checks either via ``--help`` flag or by using the autocomplete -option. - -Run the ``mypy`` check for the currently staged changes (in ``airflow/`` excluding providers): - -.. code-block:: bash - - breeze static-checks --type mypy-airflow - -Run the ``mypy`` check for all files: - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --all-files - -Run the ``ruff`` check for the ``tests/core.py`` file with verbose output: - -.. code-block:: bash - - breeze static-checks --type ruff --file tests/core.py --verbose - -Run the ``ruff`` check for the ``tests.core`` package with verbose output: - -.. code-block:: bash - - breeze static-checks --type ruff --file tests/core/* --verbose - -Run the ``ruff-format`` check for the files ``airflow/example_dags/example_bash_operator.py`` and -``airflow/example_dags/example_python_operator.py``: - -.. code-block:: bash - - breeze static-checks --type ruff-format --file airflow/example_dags/example_bash_operator.py \ - airflow/example_dags/example_python_operator.py - -Run all checks for the currently staged files: - -.. code-block:: bash - - breeze static-checks - -Run all checks for all files: - -.. code-block:: bash - - breeze static-checks --all-files - -Run all checks for last commit: - -.. code-block:: bash - - breeze static-checks --last-commit - -Run all checks for all changes in my branch since branched from main: - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --only-my-changes - -More examples can be found in -`Breeze documentation <../dev/breeze/doc/03_developer_tasks.rst#running-static-checks>`_ - - -Debugging pre-commit check scripts requiring image --------------------------------------------------- - -Those commits that use Breeze docker image might sometimes fail, depending on your operating system and -docker setup, so sometimes it might be required to run debugging with the commands. This is done via -two environment variables ``VERBOSE`` and ``DRY_RUN``. Setting them to "true" will respectively show the -commands to run before running them or skip running the commands. - -Note that you need to run pre-commit with --verbose command to get the output regardless of the status -of the static check (normally it will only show output on failure). - -Printing the commands while executing: - -.. code-block:: bash - - VERBOSE="true" pre-commit run --verbose ruff - -Just performing dry run: - -.. code-block:: bash - - DRY_RUN="true" pre-commit run --verbose ruff - ----------- Once your code passes all the static code checks, you should take a look at `Testing documentation <09_testing.rst>`__ diff --git a/contributing-docs/12_provider_distributions.rst b/contributing-docs/12_provider_distributions.rst index e7f3a5711e91b..78ab6facec96f 100644 --- a/contributing-docs/12_provider_distributions.rst +++ b/contributing-docs/12_provider_distributions.rst @@ -107,9 +107,9 @@ needs to be updated after you change dependencies: .. code:: bash - breeze static-checks --type update-providers-dependencies --all-files + prek update-providers-dependencies --all-files -If you have ``pre-commit`` installed, this will be done automatically for you when you commit the changes and +If you have ``prek`` installed, this will be done automatically for you when you commit the changes and you should do it before you make a PR with such changed dependency changes Also, you should rebuild the image ``breeze ci-image build`` or answer ``y`` when you are asked to rebuild the @@ -121,7 +121,7 @@ Provider's cross-dependencies Some of the providers have cross-dependencies with other providers distributions. This typically happens for transfer operators where operators use hooks from the other providers in case they are transferring data between the providers. The list of dependencies is maintained -(automatically with the ``update-providers-dependencies`` pre-commit) in the +(automatically with the ``update-providers-dependencies`` prek hook) in the ``generated/provider_dependencies.json``. Cross-dependencies between providers are converted into optional dependencies (extras) - if @@ -165,7 +165,7 @@ in the previous chapter. However when they are locally developed, together with of discovery of the providers is based on ``provider.yaml`` file that is placed in the top-folder of the provider. The ``provider.yaml`` is the single source of truth for the provider metadata and it is there where you should add and remove dependencies for providers (following by running -``update-providers-dependencies`` pre-commit to synchronize the dependencies with ``pyproject.toml`` +``update-providers-dependencies`` prek hook to synchronize the dependencies with ``pyproject.toml`` of Airflow). The ``provider.yaml`` file is compliant with the schema that is available in @@ -278,7 +278,7 @@ and documented. Part of the documentation is ``provider.yaml`` file ``integratio ``version`` information. This information is stripped-out from provider info available at runtime, however it is used to automatically generate documentation for the provider. -If you have pre-commits installed, pre-commit will warn you and let you know what changes need to be +If you have prek installed, it will warn you and let you know what changes need to be done in the ``provider.yaml`` file when you add a new Operator, Hooks, Sensor or Transfer. You can also take a look at the other ``provider.yaml`` files as examples. diff --git a/contributing-docs/14_metadata_database_updates.rst b/contributing-docs/14_metadata_database_updates.rst index 8d5998a7549d3..f7af5e906eaba 100644 --- a/contributing-docs/14_metadata_database_updates.rst +++ b/contributing-docs/14_metadata_database_updates.rst @@ -36,12 +36,12 @@ database schema that you have made. To generate a new migration file, run the fo Generating ~/airflow-core/src/airflow/migrations/versions/a1e23c41f123_add_new_field_to_db.py -Note that migration file names are standardized by pre-commit hook ``update-migration-references``, so that they sort alphabetically and indicate -the Airflow version in which they first appear (the alembic revision ID is removed). As a result you should expect to see a pre-commit failure +Note that migration file names are standardized by prek hook ``update-migration-references``, so that they sort alphabetically and indicate +the Airflow version in which they first appear (the alembic revision ID is removed). As a result you should expect to see a prek failure on the first attempt. Just stage the modified file and commit again (or run the hook manually before committing). -After your new migration file is run through pre-commit it will look like this: +After your new migration file is run through prek hook it will look like this: .. code-block:: @@ -70,8 +70,8 @@ To resolve these conflicts: .. code-block:: bash - pre-commit run update-migration-references --all - pre-commit run update-er-diagram --all + prek update-migration-references --all-files + prek update-er-diagram --all-files 3. Add the updated files to the staging area and continue with the rebase. diff --git a/contributing-docs/16_adding_api_endpoints.rst b/contributing-docs/16_adding_api_endpoints.rst index 2c5a154991800..74655cca4c221 100644 --- a/contributing-docs/16_adding_api_endpoints.rst +++ b/contributing-docs/16_adding_api_endpoints.rst @@ -18,7 +18,7 @@ Adding a New API Endpoint in Apache Airflow =========================================== -This documentation outlines the steps required to add a new API endpoint in Apache Airflow. It includes implementing the logic, running pre-commit checks, and documenting the changes. +This documentation outlines the steps required to add a new API endpoint in Apache Airflow. It includes implementing the logic, running prek hooks, and documenting the changes. **The outline for this document in GitHub is available at top-right corner button (with 3-dots and 3 lines).** @@ -67,19 +67,19 @@ Step 2: Add tests for your Endpoints Step 3: Documentation --------------------- -Documentation is built automatically by FastAPI and our pre-commit hooks. Verify by going to ``/docs`` that the documentation is clear and appears as expected (body and return types, query params, validation) +Documentation is built automatically by FastAPI and our prek hooks. Verify by going to ``/docs`` that the documentation is clear and appears as expected (body and return types, query params, validation) -Step 4: Run Pre-commit Hooks ------------------------------ -1. Ensure all code meets the project's quality standards by running pre-commit hooks. -2. Pre-commit hooks include static code checks, formatting, and other validations. +Step 4: Run prek Hooks +---------------------- +1. Ensure all code meets the project's quality standards by running prek hooks. +2. Prek hooks include static code checks, formatting, and other validations. 3. Persisted openapi spec is located in ``v2-rest-api-generated.yaml` and a hook will take care of updating it based on your new endpoint, you just need to add and commit the change. -4. Run the following command to execute all pre-commit hooks: +4. Run the following command to execute all prek hooks: .. code-block:: bash - pre-commit run --all-files + prek --all-files Optional: Adding Pydantic Model @@ -99,7 +99,7 @@ In some cases, you may need to define additional models for new data structures. These models are defined to structure and validate the data handled by the API. Once defined, these models will automatically be added to the OpenAPI spec file as long as they are actually used by one endpoint. -After adding or modifying Pydantic models, make sure to run the pre-commit hooks again to update any generated files. +After adding or modifying Pydantic models, make sure to run the prek hooks again to update any generated files. ------ diff --git a/contributing-docs/17_architecture_diagrams.rst b/contributing-docs/17_architecture_diagrams.rst index 73d01440642d9..f82b2a2e0c9f6 100644 --- a/contributing-docs/17_architecture_diagrams.rst +++ b/contributing-docs/17_architecture_diagrams.rst @@ -20,7 +20,7 @@ Architecture Diagrams We started to use (and gradually convert old diagrams to use it) `Diagrams `_ as our tool of choice to generate diagrams. The diagrams are generated from Python code and can be -automatically updated when the code changes. The diagrams are generated using pre-commit hooks (See +automatically updated when the code changes. The diagrams are generated using prek hooks (See static checks below) but they can also be generated manually by running the corresponding Python code. To run the code you need to install the dependencies in the virtualenv you use to run it: @@ -28,10 +28,10 @@ To run the code you need to install the dependencies in the virtualenv you use t system (``brew install graphviz`` on macOS for example). The source code of the diagrams are next to the generated diagram, the difference is that the source -code has ``.py`` extension and the generated diagram has ``.png`` extension. The pre-commit hook ``generate-airflow-diagrams`` +code has ``.py`` extension and the generated diagram has ``.png`` extension. The prek hook ``generate-airflow-diagrams`` will look for ``diagram_*.py`` files in the ``docs`` subdirectories to find them and runs them when the sources changed and the diagrams are not up to date (the -pre-commit will automatically generate an .md5sum hash of the sources and store it next to the diagram +prek hook will automatically generate an .md5sum hash of the sources and store it next to the diagram file). In order to generate the diagram manually you can run the following command: @@ -44,19 +44,19 @@ You can also generate all diagrams by: .. code-block:: bash - pre-commit run generate-airflow-diagrams + prek generate-airflow-diagrams or with Breeze: .. code-block:: bash - breeze static-checks --type generate-airflow-diagrams --all-files + prek generate-airflow-diagrams --all-files When you iterate over a diagram, you can also setup a "save" action in your IDE to run the python file automatically when you save the diagram file. Once you've done iteration and you are happy with the diagram, you can commit the diagram, the source -code and the .md5sum file. The pre-commit hook will then not run the diagram generation until the +code and the .md5sum file. The prek hook will then not run the diagram generation until the source code for it changes. ------ diff --git a/contributing-docs/18_contribution_workflow.rst b/contributing-docs/18_contribution_workflow.rst index c5308eeb4dfb2..438284c063fcb 100644 --- a/contributing-docs/18_contribution_workflow.rst +++ b/contributing-docs/18_contribution_workflow.rst @@ -40,7 +40,7 @@ In general, your contribution includes the following stages: 2. Create a `local virtualenv <07_local_virtualenv.rst>`_, initialize the `Breeze environment <../dev/breeze/doc/README.rst>`__, and - install `pre-commit framework <08_static_code_checks.rst#pre-commit-hooks>`__. + install `prek tool <08_static_code_checks.rst#prek-hooks>`__. If you want to add more changes in the future, set up your fork and enable GitHub Actions. 3. Join `devlist `__ @@ -176,9 +176,9 @@ Step 4: Prepare PR * Modify the class and add necessary code and unit tests. * Run and fix all the `static checks <08_static_code_checks.rst>`__. If you have - `pre-commits installed <08_static_code_checks.rst#pre-commit-hooks>`__, + `prek installed <08_static_code_checks.rst#prek-hooks>`__, this step is automatically run while you are committing your code. If not, you can do it manually - via ``git add`` and then ``pre-commit run``. + via ``git add`` and then ``prek``. * Run the appropriate tests as described in `Testing documentation <09_testing.rst>`__. diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index 401b383ab3bc9..a5d6c6deb873e 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -298,9 +298,6 @@ Then, create the integration test file under ``tests/integration`` - remember to and to use the ``@pytest.mark.integration`` decorator. It is recommended to define setup and teardown methods (``setup_method`` and ``teardown_method``, respectively) - you could look at existing integration tests to learn more. -Before pushing to GitHub, make sure to run static checks (``breeze static-checks --only-my-changes``) to apply linters -on the Python logic, as well as to update the commands images under ``dev/breeze/docs/images``. - When writing integration tests for components that also require Kerberos, you could enforce auto-enabling the latter by updating ``compose_file()`` method in ``airflow_breeze.params.shell_params.ShellParams``. For example, to ensure that Kerberos is active for ``trino`` integration tests, the following code has been introduced: diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 6cdee1cc6302d..c1186c987e9db 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -308,7 +308,7 @@ uv tool install -e ./dev/breeze - Set your version in `airflow/__init__.py` (without the RC tag). - Run `git commit` without a message to update versions in `docs`. -- Add supported Airflow version to `./scripts/ci/pre_commit/supported_versions.py` and let pre-commit do the job again. +- Add supported Airflow version to `./scripts/ci/prek/supported_versions.py` and let prek do the job again. - Replace the versions in `README.md` about installation and verify that installation instructions work fine. - Add entry for default python version to `BASE_PROVIDERS_COMPATIBILITY_CHECKS` in `src/airflow_breeze/global_constants.py` with the new Airflow version, and empty exclusion for providers. This list should be updated later when providers @@ -1142,7 +1142,7 @@ EOF This includes: -- Modify `./scripts/ci/pre_commit/supported_versions.py` and let pre-commit do the job. +- Modify `./scripts/ci/prek/supported_versions.py` and let prek do the job. - For major/minor release, update version in `airflow/__init__.py` and `docs/docker-stack/` to the next likely minor version release. - Sync `RELEASE_NOTES.rst` (including deleting relevant `newsfragments`) and `README.md` changes. - Updating `Dockerfile` with the new version. diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index 3250197a12edf..eccf224ae98a2 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -156,7 +156,7 @@ annotations: ``` Make sure that all the release notes changes are submitted as PR and merged. Changes in release notes should -also automatically (via `pre-commit` trigger updating of the [reproducible_build.yaml](../chart/reproducible_build.yaml)) +also automatically (via `prek` trigger updating of the [reproducible_build.yaml](../chart/reproducible_build.yaml)) file which is uses to reproducibly build the chart package and source tarball. You can leave the k8s environment now: diff --git a/dev/breeze/README.md b/dev/breeze/README.md index b80e3a60c3db2..3eef940fdf620 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -131,10 +131,10 @@ Note that when you update dependencies/lock them you should commit the changes i See [uv documentation](https://docs.astral.sh/uv/getting-started/) for more details on using `uv`. -PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. +PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PREK. --------------------------------------------------------------------------------------------------------- -Package config hash: ca0dde3dc813baf255b7f645c73025b9ce0729d24acf33499f15ba62a6d94cb6e665fa84c575fc88b5a20f3a1c73ded735ff2dcfad353d2b91434b03e11a4467 +Package config hash: 4c796051a27cfdb7f49a758f6c306f75e606c6bbe66d0db79c8be74a88fd5636255abe6930cb26af4b48bfbe81b4bed303738070825cf5b456c9bd843c92f03c --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/01_installation.rst b/dev/breeze/doc/01_installation.rst index 5f0acd5e080f8..f1aa42f5ecb5c 100644 --- a/dev/breeze/doc/01_installation.rst +++ b/dev/breeze/doc/01_installation.rst @@ -125,7 +125,7 @@ Docker in WSL 2 If you're experiencing errors such as ``ERROR: for docker-compose_airflow_run Cannot create container for service airflow: not a directory`` when starting Breeze after the first time or an error like ``docker: Error response from daemon: not a directory. - See 'docker run --help'.`` when running the pre-commit tests, you may need to consider + See 'docker run --help'.`` when running the prek tests, you may need to consider `installing Docker directly in WSL 2 `_ instead of using Docker Desktop for Windows. diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index fd8530684b473..c7d0c006037d5 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -265,32 +265,30 @@ as the short hand operator. Running static checks --------------------- -You can run static checks via Breeze. You can also run them via pre-commit command but with auto-completion -Breeze makes it easier to run selective static checks. If you press after the static-check and if -you have auto-complete setup you should see auto-completable list of all checks available. +You can run static checks via prek. For example, this following command: .. code-block:: bash - breeze static-checks --type mypy-airflow + prek mypy-airflow will run mypy check for currently staged files inside ``airflow/`` excluding providers. Selecting files to run static checks on --------------------------------------- -Pre-commits run by default on staged changes that you have locally changed. It will run it on all the +Prek hooks run by default on staged changes that you have locally changed. It will run it on all the files you run ``git add`` on and it will ignore any changes that you have modified but not staged. If you want to run it on all your modified files you should add them with ``git add`` command. With ``--all-files`` you can run static checks on all files in the repository. This is useful when you want to be sure they will not fail in CI, or when you just rebased your changes and want to -re-run latest pre-commits on your changes, but it can take a long time (few minutes) to wait for the result. +re-run latest prek hooks on your changes, but it can take a long time (few minutes) to wait for the result. .. code-block:: bash - breeze static-checks --type mypy-airflow --all-files + prek mypy-airflow --all-files The above will run mypy check for all files. @@ -299,58 +297,35 @@ specifying (can be multiple times) ``--file`` flag. .. code-block:: bash - breeze static-checks --type mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py + prek mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py The above will run mypy check for those to files (note: autocomplete should work for the file selection). However, often you do not remember files you modified and you want to run checks for files that belong -to specific commits you already have in your branch. You can use ``breeze static check`` to run the checks +to specific commits you already have in your branch. You can use ``prek`` to run the checks only on changed files you have already committed to your branch - either for specific commit, for last commit, for all changes in your branch since you branched off from main or for specific range of commits you choose. .. code-block:: bash - breeze static-checks --type mypy-airflow --last-commit + prek mypy-airflow --last-commit The above will run mypy check for all files in the last commit in your branch. .. code-block:: bash - breeze static-checks --type mypy-airflow --only-my-changes - -The above will run mypy check for all commits in your branch which were added since you branched off from main. - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --commit-ref 639483d998ecac64d0fef7c5aa4634414065f690 - -The above will run mypy check for all files in the 639483d998ecac64d0fef7c5aa4634414065f690 commit. -Any ``commit-ish`` reference from Git will work here (branch, tag, short/long hash etc.) - -.. code-block:: bash - - breeze static-checks --type identity --verbose --from-ref HEAD^^^^ --to-ref HEAD + prek identity --verbose --from-ref HEAD^^^^ --to-ref HEAD The above will run the check for the last 4 commits in your branch. You can use any ``commit-ish`` references in ``--from-ref`` and ``--to-ref`` flags. - -These are all available flags of ``static-checks`` command: - -.. image:: ./images/output_static-checks.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_static-checks.svg - :width: 100% - :alt: Breeze static checks - - .. note:: When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume so that it can speed up static checks execution significantly. However, sometimes, the cache might get broken, in which case you should run ``breeze down`` to clean up the cache. - .. note:: You cannot change Python version for static checks that are run within Breeze containers. @@ -440,7 +415,7 @@ command takes care about it. This is needed when you want to run API server insi Note -This command requires the ``pre-commit`` tool, which should be installed by following `this guide <../../../contributing-docs/03_contributors_quick_start.rst#configuring-pre-commit>`__. +This command requires the ``prek`` tool, which should be installed by following `this guide <../../../contributing-docs/03_contributors_quick_start.rst#configuring-prek>`__. Breeze cleanup -------------- diff --git a/dev/breeze/doc/04_troubleshooting.rst b/dev/breeze/doc/04_troubleshooting.rst index 9f7a409b01f95..5354986f5e6a6 100644 --- a/dev/breeze/doc/04_troubleshooting.rst +++ b/dev/breeze/doc/04_troubleshooting.rst @@ -99,30 +99,28 @@ or ERROR: Package 'blacken-docs' requires a different Python: 3.8.18 not in '>=3.10' -It means that your pre-commit hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall -it and clean pre-commit cache. +It means that your prek hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall +it and clean prek cache. -This can be done with ``uv tool`` to install ``pre-commit``) +This can be done with ``uv tool`` to install ``prek``) .. code-block:: bash - uv tool uninstall pre-commit - uv tool install pre-commit --python 3.10 --force --with pre-commit-uv - pre-commit clean - pre-commit install + uv tool uninstall prek + uv tool install prek --python 3.10 --force + prek clean + prek install You can also use ``pipx`` .. code-block:: bash - pipx uninstall pre-commit - pipx install pre-commit --python $(which python3.10) --force - # This one allows pre-commit to use uv for venvs installed by pre-commit - pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to install virtualenvs - pre-commit clean - pre-commit install + pipx uninstall prek + pipx install prek --python $(which python3.10) --force + prek clean + prek install -If you installed ``pre-commit`` differently, you should remove and reinstall +If you installed ``prek`` differently, you should remove and reinstall it (and clean cache) following the way you installed it. @@ -132,7 +130,7 @@ Bad Interpreter Error with ``pipx`` If you are experiencing bad interpreter errors ``zsh: /Users/eladkal/.local/bin/breeze: bad interpreter: /Users/eladkal/.local/pipx/venvs/apache-airflow-breeze/bin/python: no such file or directory`` -try to run ``pipx list`` to view which packages has bad interpreter (it can be more than just breeze, for example pre-commit) +try to run ``pipx list`` to view which packages has bad interpreter (it can be more than just breeze, for example prek) you can fix these errors by running ``pipx reinstall-all`` ETIMEDOUT Error @@ -170,7 +168,7 @@ When running ``breeze start-airflow``, either normally or in ``dev-mode``, the f The asset compilation failed. Exiting. - [INFO] Locking pre-commit directory + [INFO] Locking prek directory Error 1 returned diff --git a/dev/breeze/doc/07_breeze_maintenance_tasks.rst b/dev/breeze/doc/07_breeze_maintenance_tasks.rst index f139040375ed0..83f502106d60a 100644 --- a/dev/breeze/doc/07_breeze_maintenance_tasks.rst +++ b/dev/breeze/doc/07_breeze_maintenance_tasks.rst @@ -29,7 +29,7 @@ Regenerating documentation SVG screenshots This documentation contains exported SVG screenshots with "help" of their commands and parameters. You can regenerate those images that need to be regenerated because their commands changed (usually after the breeze code has been changed) via ``regenerate-command-images`` command. Usually this is done -automatically via pre-commit, but sometimes (for example when ``rich`` or ``rich-click`` library changes) +automatically via ``prek``, but sometimes (for example when ``rich`` or ``rich-click`` library changes) you need to regenerate those images. You can add ``--force`` flag (or ``FORCE="true"`` environment variable to regenerate all images (not diff --git a/dev/breeze/doc/10_advanced_breeze_topics.rst b/dev/breeze/doc/10_advanced_breeze_topics.rst index d3f393c2b3963..abee7e0acdf03 100644 --- a/dev/breeze/doc/10_advanced_breeze_topics.rst +++ b/dev/breeze/doc/10_advanced_breeze_topics.rst @@ -149,16 +149,13 @@ If you want to add core dependency that should always be installed - you need to to ``dependencies`` section. If you want to add it to one of the optional core extras, you should add it in the extra definition in ``pyproject.toml`` (you need to find out where it is defined). If you want to add it to one of the providers, you need to add it to the ``provider.yaml`` file in the provider -directory - but remember that this should be followed by running pre-commit that will automatically update +directory - but remember that this should be followed by running prek that will automatically update the ``pyproject.toml`` with the new dependencies as the ``provider.yaml`` files are not used directly, they are used to update ``pyproject.toml`` file: .. code-block:: bash - pre-commit run update-providers-dependencies --all-files - -You can also run the pre-commit by ``breeze static-checks --type update-providers-dependencies --all-files`` -command - which provides autocomplete. + prek update-providers-dependencies --all-files After you've updated the dependencies, you need to rebuild the image: @@ -191,15 +188,11 @@ scripts are present in ``scripts/docker`` directory and are aptly (!) named ``in of the apt dependencies are installed in the ``install_os_dependencies.sh``, but some are installed in other scripts (for example ``install_postgres.sh`` or ``install_mysql.sh``). -After you modify the dependencies in the scripts, you need to inline them by running pre-commit: +After you modify the dependencies in the scripts, you need to inline them by running prek: .. code-block:: bash - pre-commit run update-inlined-dockerfile-scripts --all-files - -You can also run the pre-commit by ``breeze static-checks --type update-inlined-dockerfile-scripts --all-files`` -command - which provides autocomplete. - + prek update-inlined-dockerfile-scripts --all-files After you've updated the dependencies, you need to rebuild the image: diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 05f0748996572..41edea67d6431 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -443,9 +443,8 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `25.2` | `pip` version used. | -| `AIRFLOW_UV_VERSION` | `0.8.9` | `uv` version used. | -| `AIRFLOW_PRE_COMMIT_VERSION` | `4.3.0` | `pre-commit` version used. | -| `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | +| `AIRFLOW_UV_VERSION` | `0.8.11` | `uv` version used. | +| `AIRFLOW_PREK_VERSION` | `0.0.26` | `prek` version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 60d9966a20c51..8f6f255985cf9 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -24,7 +24,7 @@ - [Selective CI Checks](#selective-ci-checks) - [Groups of files that selective check make decisions on](#groups-of-files-that-selective-check-make-decisions-on) - [Selective check decision rules](#selective-check-decision-rules) - - [Skipping pre-commits (Static checks)](#skipping-pre-commits-static-checks) + - [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks) - [Suspended providers](#suspended-providers) - [Selective check outputs](#selective-check-outputs) - [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs) @@ -111,7 +111,7 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel * If there are no files left in sources after matching the test types and Kubernetes files, then apparently some Core/Other files have been changed. This automatically adds all test types to execute. This is done because changes in core might impact all the other test types. -* if `CI Image building` is disabled, only basic pre-commits are enabled - no 'image-depending` pre-commits +* if `CI Image building` is disabled, only basic prek hooks are enabled - no 'image-depending` prek hooks are enabled. * If there are some build dependencies changed (`hatch_build.py` and updated system dependencies in the `pyproject.toml` - then `upgrade to newer dependencies` is enabled. @@ -121,12 +121,12 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel changed, also providers docs are built because all providers depend on airflow docs. If any of the docs build python files changed or when build is "canary" type in main - all docs packages are built. -## Skipping pre-commits (Static checks) +## Skipping prek hooks (Static checks) -Our CI always run pre-commit checks with `--all-files` flag. This is in order to avoid cases where -different check results are run when only subset of files is used. This has an effect that the pre-commit +Our CI always run prek checks with `--all-files` flag. This is in order to avoid cases where +different check results are run when only subset of files is used. This has an effect that the prek tests take a long time to run when all of them are run. Selective checks allow to save a lot of time -for those tests in regular PRs of contributors by smart detection of which pre-commits should be skipped +for those tests in regular PRs of contributors by smart detection of which prek hooks should be skipped when some files are not changed. Those are the rules implemented: * The `identity` check is always skipped (saves space to display all changed files in CI) @@ -137,7 +137,7 @@ when some files are not changed. Those are the rules implemented: * check-provider-yaml-valid * lint-helm-chart * mypy-providers -* If "full tests" mode is detected, no more pre-commits are skipped - we run all of them +* If "full tests" mode is detected, no more prek hooks are skipped - we run all of them * The following checks are skipped if those files are not changed: * if no `All Providers Python files` changed - `mypy-providers` check is skipped * if no `All Airflow Python files` changed - `mypy-airflow` check is skipped @@ -230,7 +230,7 @@ GitHub Actions to pass the list of parameters to a command to execute | amd-runners | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | | | arm-runners | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | | | selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | -| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | +| skip-prek-hooks | Which prek hooks should be skipped during the static-checks run | flynt,identity | | | skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | | sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | | testable-core-integrations | List of core integrations that are testable in the build as JSON array | \['celery', 'kerberos'\] | | @@ -244,7 +244,7 @@ When we decided whether to run `full tests` we do not check (directly) if provid even if they are single source of truth for provider dependencies and when you add a dependency there, the environment changes and generally full tests are advised. -This is because provider.yaml change will automatically trigger (via `update-provider-dependencies` pre-commit) +This is because provider.yaml change will automatically trigger (via `update-provider-dependencies` prek) generation of `generated/provider_dependencies.json` and `pyproject.toml` gets updated as well. This is a far better indication if we need to run full tests than just checking if provider.yaml files changed, because provider.yaml files contain more information than just dependencies - they are the single source of truth diff --git a/dev/breeze/doc/ci/05_workflows.md b/dev/breeze/doc/ci/05_workflows.md index 8db97a96a8dea..ed7211a3e6125 100644 --- a/dev/breeze/doc/ci/05_workflows.md +++ b/dev/breeze/doc/ci/05_workflows.md @@ -202,7 +202,7 @@ Windows. they are needed (basically, when Python code has been modified). `(6)` On non-main builds some of the static checks that are related to -Providers are skipped via selective checks (`skip-pre-commits` check). +Providers are skipped via selective checks (`skip-prek-hooks` check). `(7)` On non-main builds the unit tests, docs and integration tests for providers are skipped via selective checks. diff --git a/dev/breeze/doc/ci/07_running_ci_locally.md b/dev/breeze/doc/ci/07_running_ci_locally.md index d71cddac74adc..2bc268bfb6441 100644 --- a/dev/breeze/doc/ci/07_running_ci_locally.md +++ b/dev/breeze/doc/ci/07_running_ci_locally.md @@ -36,7 +36,7 @@ The main goal of the CI philosophy we have that no matter how complex the test and integration infrastructure, as a developer you should be able to reproduce and re-run any of the failed checks locally. One part -of it are pre-commit checks, that allow you to run the same static +of it are prek checks, that allow you to run the same static checks in CI and locally, but another part is the CI environment which is replicated locally with Breeze. @@ -163,8 +163,8 @@ Those variables are used to control the initialization of the environment in the | Variable | Option | Local dev | CI | Comment | |---------------------------------|------------------------------------|-----------|-----------|-----------------------------------------------------------------------------| | MOUNT_SOURCES | --mount-sources | | skip | Whether to mount the local sources into the container. | -| SKIP_ENVIRONMENT_INITIALIZATION | --skip-environment-initialization | false (*) | false (*) | Skip initialization of test environment (*) set to true in pre-commits. | -| SKIP_IMAGE_UPGRADE_CHECK | --skip-image-upgrade-check | false (*) | false (*) | Skip checking if image should be upgraded (*) set to true in pre-commits. | +| SKIP_ENVIRONMENT_INITIALIZATION | --skip-environment-initialization | false (*) | false (*) | Skip initialization of test environment (*) set to true in prek hooks. | +| SKIP_IMAGE_UPGRADE_CHECK | --skip-image-upgrade-check | false (*) | false (*) | Skip checking if image should be upgraded (*) set to true in prek hooks. | | SKIP_PROVIDERS_TESTS | | false | false | Skip running provider integration tests (in non-main branch). | | SKIP_SSH_SETUP | | false | false (*) | Skip setting up SSH server for tests. (*) set to true in GitHub CodeSpaces. | | VERBOSE_COMMANDS | | false | false | Whether every command executed in docker should be printed. | diff --git a/dev/breeze/doc/images/output-commands-hash.txt b/dev/breeze/doc/images/output-commands-hash.txt index 407f5d55ae7f0..c27896f7b526b 100644 --- a/dev/breeze/doc/images/output-commands-hash.txt +++ b/dev/breeze/doc/images/output-commands-hash.txt @@ -1,4 +1,4 @@ -# This file is automatically generated by pre-commit. If you have a conflict with this file +# This file is automatically generated by prek. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:ffb1a766b791beaf5f8a983587db870f diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index fb3f3278b25ac..d3656bc530f4e 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +