diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58de99f71c3c0..068215b380c56 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,13 +189,6 @@ repos: language: python pass_filenames: false require_serial: true - - id: check-taskinstance-tis-attrs - name: Check that TI and TIS have the same attributes - entry: ./scripts/ci/prek/check_ti_vs_tis_attributes.py - language: python - files: ^airflow-core/src/airflow/models/taskinstance\.py$|^airflow-core/src/airflow/models/taskinstancehistory\.py$ - pass_filenames: false - require_serial: true - repo: https://github.com/adamchainz/blacken-docs rev: dda8db18cfc68df532abf33b185ecd12d5b7b326 # frozen: 1.20.0 hooks: @@ -379,62 +372,18 @@ repos: files: Dockerfile.*$ pass_filenames: true require_serial: true - - id: check-airflow-k8s-not-used - name: Check airflow.kubernetes imports are not used - language: python - files: ^airflow-core/src/airflow/.*\.py$ - require_serial: true - exclude: ^airflow-core/src/airflow/kubernetes/ - entry: ./scripts/ci/prek/check_airflow_imports.py - --pattern '^airflow\.kubernetes' - --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." - - id: check-common-compat-used-for-openlineage - name: Check common.compat is used for OL deprecated classes - language: python - files: ^airflow-core/src/airflow/.*\.py$ - require_serial: true - exclude: > - (?x) - ^airflow-core/src/airflow/openlineage/| - ^airflow/providers/common/compat/openlineage/facet.py$ - entry: ./scripts/ci/prek/check_airflow_imports.py - --pattern '^openlineage\.client\.(facet|run)' - --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." - id: check-airflow-providers-bug-report-template name: Sort airflow-bug-report provider list language: python files: ^\.github/ISSUE_TEMPLATE/3-airflow_providers_bug_report\.yml$ require_serial: true entry: ./scripts/ci/prek/check_airflow_bug_report_template.py - - id: check-cncf-k8s-only-for-executors - name: Check cncf.kubernetes imports used for executors only - language: python - files: ^airflow-core/src/airflow/.*\.py$ - require_serial: true - exclude: > - (?x) - ^providers/.*/src/airflow/providers/| - ^airflow-core/src/airflow/exceptions\.py$| - ^airflow-core/src/airflow/models/renderedtifields\.py$| - ^airflow-core/src/airflow/serialization/serialized_objects\.py$| - ^airflow-core/src/airflow/serialization/serializers/kubernetes\.py$| - ^airflow-core/src/airflow/utils/sqlalchemy\.py$ - entry: ./scripts/ci/prek/check_airflow_imports.py - --pattern '^airflow\.providers\.cncf\.kubernetes' - --message "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." - id: update-local-yml-file name: Update mounts in the local yml file entry: ./scripts/ci/prek/local_yml_mounts.py language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false - - id: check-extra-packages-references - name: Checks setup extra packages - description: Checks if all the extras defined in hatch_build.py are listed in extra-packages-ref.rst file - language: python - files: ^airflow-core/docs/extra-packages-ref\.rst$|^hatch_build\.py$ - pass_filenames: false - entry: ./scripts/ci/prek/check_extra_packages_ref.py - id: check-extras-order name: Check order of extras in Dockerfile entry: ./scripts/ci/prek/check_order_dockerfile_extras.py @@ -497,13 +446,6 @@ repos: entry: "\\|\\s*safe" files: \.html$ pass_filenames: true - - id: check-no-providers-in-core-examples - language: pygrep - name: No providers imports in core example DAGs - description: The core example DAGs have no dependencies other than standard provider or core Airflow - entry: "^\\s*from airflow\\.providers.(?!standard.)" - pass_filenames: true - files: ^airflow-core/src/airflow/example_dags/.*\.py$ - id: check-urlparse-usage-in-code language: pygrep name: Don't use urlparse in code @@ -511,13 +453,6 @@ repos: entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$" pass_filenames: true files: \.py$ - - id: check-only-new-session-with-provide-session - name: Check NEW_SESSION is only used with @provide_session - language: python - entry: ./scripts/ci/prek/new_session_in_provide_session.py - pass_filenames: true - files: ^airflow-core/src/airflow/.+\.py$ - exclude: ^airflow-core/src/airflow/serialization/pydantic/.* - id: check-for-inclusive-language language: pygrep name: Check for language that we do not accept as community @@ -611,17 +546,15 @@ repos: ^scripts/ci/docker-compose/integration-keycloak\.yml$| ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$| ^scripts/ci/prek/vendor_k8s_json_schema\.py$ - - id: check-base-operator-partial-arguments - name: Check BaseOperator and partial() arguments - language: python - entry: ./scripts/ci/prek/check_base_operator_partial_arguments.py - pass_filenames: false - files: ^airflow-core/src/airflow/models/(?:base|mapped)operator\.py$ - id: check-template-context-variable-in-sync name: Sync template context variable refs language: python entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py - files: ^airflow-core/src/airflow/models/taskinstance\.py$|^task-sdk/src/airflow/sdk/definitions/context\.py$|^airflow-core/docs/templates-ref\.rst$ + files: + (?x) + ^airflow-core/src/airflow/models/taskinstance\.py$| + ^task-sdk/src/airflow/sdk/definitions/context\.py$| + ^airflow-core/docs/templates-ref\.rst$ - id: check-base-operator-usage language: pygrep name: Check BaseOperator core imports @@ -649,12 +582,6 @@ repos: ^airflow-core/src/airflow/operators/.*$| ^providers/.*/src/airflow/providers/.*$| ^providers/.*/src/airflow/providers/standard/sensors/.*$ - - id: check-decorated-operator-implements-custom-name - name: Check @task decorator implements custom_operator_name - language: python - entry: ./scripts/ci/prek/decorator_operator_implements_custom_name.py - pass_filenames: true - files: ^airflow-core/src/airflow/.*\.py$ - id: check-core-deprecation-classes language: pygrep name: Verify usage of Airflow deprecation classes in core @@ -720,6 +647,8 @@ repos: entry: koalaman/shellcheck:v0.8.0 -x -a files: \.(bash|sh)$|^hooks/build$|^hooks/push$ exclude: ^dev/breeze/autocomplete/.*$ + # Note: Actually the compile-ui-assets prek hook would be best in airflow-core/.pre-commit-config.yaml + # See also https://github.com/j178/prek/issues/973 - id: compile-ui-assets name: Compile ui assets (manual) language: node @@ -729,6 +658,7 @@ repos: entry: ./scripts/ci/prek/compile_ui_assets.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] + # Note: Keeping compile-ui-assets-dev together with ^^^ - id: compile-ui-assets-dev name: Compile ui assets in dev mode (manual) language: node @@ -771,13 +701,6 @@ repos: files: ^dev/breeze/.*$ pass_filenames: false require_serial: true - - id: check-tests-in-the-right-folders - name: Check if tests are in the right folders - entry: ./scripts/ci/prek/check_tests_in_right_folders.py - language: python - files: ^airflow-core/tests/.*\.py$ - pass_filenames: true - require_serial: true - id: check-system-tests-present name: Check if system tests have required segments of code entry: ./scripts/ci/prek/check_system_tests.py @@ -833,16 +756,6 @@ repos: ^scripts/ci/docker-compose/gremlin/.| ^scripts/ci/docker-compose/.+-config\.ya?ml$ require_serial: true - - id: lint-json-schema - name: Lint config_templates/config.yml - entry: ./scripts/ci/prek/lint_json_schema.py - args: - - --spec-file - - airflow-core/src/airflow/config_templates/config.yml.schema.json - language: python - pass_filenames: true - files: ^airflow-core/src/airflow/config_templates/config\.yml$ - require_serial: true - id: check-persist-credentials-disabled-in-github-workflows name: Check persistent creds in workflow files description: Check that workflow files have persist-credentials disabled @@ -864,12 +777,6 @@ repos: Zip files are not allowed in the repository as they are hard to track and have security implications. Please remove the zip file from the repository. files: \.zip$ - - id: check-code-deprecations - name: Check deprecations categories in decorators - entry: ./scripts/ci/prek/check_deprecations.py - language: python - pass_filenames: true - files: ^airflow-core/src/airflow/.*\.py$ - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts entry: ./scripts/ci/prek/inline_scripts_in_docker.py @@ -898,16 +805,6 @@ repos: # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes # This is fast, so not too much downside always_run: true - - id: check-significant-newsfragments-are-valid - name: Check significant newsfragments are valid - # Significant newsfragments follows a special format so that we can group information easily. - language: python - files: ^airflow-core/newsfragments/.*\.rst$ - entry: ./scripts/ci/prek/significant_newsfragments_checker.py - pass_filenames: false - # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes - # This is fast, so not too much downside - always_run: true - id: update-breeze-cmd-output name: Update breeze docs description: Update output of breeze commands in Breeze documentation @@ -926,7 +823,11 @@ repos: entry: ./scripts/ci/prek/update_example_dags_paths.py language: python pass_filenames: true - files: ^airflow-core/docs/.*example-dags\.rst$|^docs/.*index\.rst$^airflow-core/docs/.*index\.rst$ + files: + (?x) + ^airflow-core/docs/.*example-dags\.rst$| + ^airflow-core/docs/.*index\.rst$| + ^docs/.*index\.rst$ always_run: true - id: check-lazy-logging name: Check that all logging methods are lazy @@ -934,19 +835,6 @@ repos: language: python pass_filenames: true files: \.py$ - - id: create-missing-init-py-files-tests - name: Create missing init.py files in tests - entry: ./scripts/ci/prek/check_init_in_tests.py - language: python - pass_filenames: false - files: ^airflow-core/tests/.*\.py$ - - id: check-tests-unittest-testcase - name: Unit tests do not inherit from unittest.TestCase - description: Check that unit tests do not inherit from unittest.TestCase - entry: ./scripts/ci/prek/unittest_testcase.py - language: python - pass_filenames: true - files: ^airflow-core/tests/.*\.py$ - id: bandit name: bandit description: "Bandit is a tool for finding common security issues in Python code" @@ -956,7 +844,7 @@ repos: types: [python] additional_dependencies: ['bandit==1.7.6'] require_serial: true - files: ^airflow-core/src/airflow/.* + files: ^airflow-core/src/airflow/.* # TODO Expand this to more than just airflow-core exclude: airflow/example_dags/.* args: @@ -964,64 +852,6 @@ repos: - "B101,B301,B324,B403,B404,B603" - "--severity-level" - "high" # TODO: remove this line when we fix all the issues - - id: check-no-fab-migrations - language: pygrep - name: Check no migration is done on FAB related table - description: > - FAB tables are no longer used in core Airflow but in FAB provider. - As such, it is forbidden to create migrations related to FAB tables in core Airflow. - Such migrations should be in FAB provider. To achieve this, a new capability must be implemented: - support migrations for providers. In other words, providers need to be able to specify migrations - so that, any FAB related migration (besides the legacy ones) is defined in FAB provider. - See https://github.com/apache/airflow/issues/32210 - entry: > - (?ix) - \bab_permission\b| - \bab_view_menu\b| - \bab_role\b| - \bab_permission_view\b| - \bab_permission_view_role\b| - \bab_user\b| - \bab_user_role\b| - \bab_register_user\b - pass_filenames: true - files: ^airflow-core/src/airflow/migrations/versions/.*\.py$ - exclude: - ^airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py$ - - id: ts-compile-lint-ui - name: Compile / format / lint UI - description: TS types generation / ESLint / Prettier new UI files - language: node - files: | - (?x) - ^airflow-core/src/airflow/ui/.*\.(js|ts|tsx|yaml|css|json)$| - ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ - exclude: | - (?x) - ^airflow-core/src/airflow/ui/node-modules/.*| - ^airflow-core/src/airflow/ui/.pnpm-store - entry: ./scripts/ci/prek/ts_compile_lint_ui.py - additional_dependencies: ['pnpm@9.7.1'] - pass_filenames: true - require_serial: true - - id: ts-compile-lint-simple-auth-manager-ui - name: Compile / format / lint simple auth manager UI - description: TS types generation / ESLint / Prettier new UI files - language: node - files: | - (?x) - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/.*\.(js|ts|tsx|yaml|css|json)$| - ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/.*\.yaml$ - exclude: | - (?x) - ^airflow-core/src/airflow/api_fastapi/node-modules/.*| - ^airflow-core/src/airflow/api_fastapi/.pnpm-store - entry: ./scripts/ci/prek/ts_compile_lint_simple_auth_manager_ui.py - additional_dependencies: ['pnpm@9.7.1'] - pass_filenames: true - require_serial: true ## ADD MOST PREK HOOK ABOVE THAT LINE # The below prek hooks are those requiring CI image to be built - id: mypy-dev @@ -1039,21 +869,6 @@ repos: pass_filenames: false files: ^.*\.py$ require_serial: true - - id: mypy-airflow-core - stages: ['pre-push'] - name: Run mypy for airflow-core - language: python - entry: ./scripts/ci/prek/mypy.py - files: ^airflow-core/.*\.py$ - require_serial: true - - id: mypy-airflow-core - stages: ['manual'] - name: Run mypy for airflow-core (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py airflow-core - pass_filenames: false - files: ^airflow-core/.*\.py$ - require_serial: true - id: mypy-devel-common stages: ['pre-push'] name: Run mypy for devel-common @@ -1069,32 +884,12 @@ repos: pass_filenames: false files: ^.*\.py$ require_serial: true - - id: generate-openapi-spec - name: Generate the FastAPI API spec - language: python - entry: ./scripts/ci/prek/generate_openapi_spec.py - pass_filenames: false - files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$ - exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* - - id: check-i18n-json - name: Check i18n files validity - description: Check i18n files are valid json, have no TODOs, and auto-format them - language: python - files: ^airflow-core/src/airflow/ui/public/i18n/locales/.*\.json$ - entry: ./scripts/ci/prek/check_i18n_json.py - pass_filenames: false - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python entry: ./scripts/ci/prek/check_template_fields.py files: ^(providers/.*/)?airflow-core/.*/(sensors|operators)/.*\.py$ require_serial: true - - id: update-migration-references - name: Update migration ref doc - language: python - entry: ./scripts/ci/prek/migration_reference.py - pass_filenames: false - files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ - id: generate-tasksdk-datamodels name: Generate Datamodels for TaskSDK client language: python @@ -1110,143 +905,12 @@ repos: uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen && uv run -p 3.12 --no-dev --no-progress --active --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen --input="../airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v2-simple-auth-manager-generated.yaml" --output="src/airflowctl/api/datamodels/auth_generated.py"' pass_filenames: false - files: ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/(datamodels|routes|services|openapi)/.*\.py$ - require_serial: true - - id: update-er-diagram - name: Update ER diagram - language: python - entry: ./scripts/ci/prek/update_er_diagram.py - pass_filenames: false - files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ - - id: check-default-configuration - name: Check the default configuration - entry: ./scripts/ci/prek/check_default_configuration.py - language: python - require_serial: true - pass_filenames: false - files: ^airflow-core/src/airflow/config_templates/config\.yml$ - - id: check-airflow-version-checks-in-core - language: pygrep - name: No AIRFLOW_V_* imports in airflow-core - entry: "import AIRFLOW_V_" - files: ^airflow-core/.*\.py$ - pass_filenames: true - # TODO (@amoghrajesh): revisit last few in this list as they all rely on versioned secrets masker imports - exclude: > - (?x) - ^airflow-core/tests/integration/otel/dags/otel_test_dag_with_pause_between_tasks\.py$| - ^airflow-core/tests/integration/otel/dags/otel_test_dag_with_pause_in_task\.py$| - ^airflow-core/tests/integration/otel/test_otel\.py$| - ^airflow-core/tests/unit/core/test_configuration\.py$| - ^airflow-core/tests/unit/models/test_renderedtifields\.py$| - ^airflow-core/tests/unit/models/test_variable\.py$ - - id: check-sdk-imports - name: Check for SDK imports in core files - entry: ./scripts/ci/prek/check_sdk_imports.py - language: python - types: [python] - files: ^airflow-core/src/airflow/ - exclude: | + files: (?x) - # Allow SDK imports in these legitimate locations - ^airflow-core/src/airflow/example_dags/.*\.py$| - - # TODO: These files need to be refactored to remove SDK coupling - ^airflow-core/src/airflow/__init__\.py$| - ^airflow-core/src/airflow/api/common/mark_tasks\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/assets\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/connections\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/hitl\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/routes/ui/structure\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/services/public/connections\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/services/ui/connections\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid\.py$| - ^airflow-core/src/airflow/api_fastapi/core_api/services/ui/task_group.py$| - ^airflow-core/src/airflow/api_fastapi/execution_api/routes/hitl\.py$| - ^airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances\.py$| - ^airflow-core/src/airflow/api_fastapi/logging/decorators\.py$| - ^airflow-core/src/airflow/assets/evaluation\.py$| - ^airflow-core/src/airflow/assets/manager\.py$| - ^airflow-core/src/airflow/cli/commands/connection_command\.py$| - ^airflow-core/src/airflow/cli/commands/task_command\.py$| - ^airflow-core/src/airflow/cli/commands/triggerer_command.py$| - ^airflow-core/src/airflow/configuration\.py$| - ^airflow-core/src/airflow/dag_processing/collection\.py$| - ^airflow-core/src/airflow/dag_processing/manager\.py$| - ^airflow-core/src/airflow/dag_processing/processor\.py$| - ^airflow-core/src/airflow/dag_processing/dagbag\.py$| - ^airflow-core/src/airflow/datasets/metadata\.py$| - ^airflow-core/src/airflow/exceptions\.py$| - ^airflow-core/src/airflow/executors/local_executor\.py$| - ^airflow-core/src/airflow/jobs/triggerer_job_runner\.py$| - ^airflow-core/src/airflow/lineage/hook\.py$| - ^airflow-core/src/airflow/listeners/spec/asset\.py$| - ^airflow-core/src/airflow/listeners/spec/taskinstance\.py$| - ^airflow-core/src/airflow/logging/remote\.py$| - ^airflow-core/src/airflow/models/__init__\.py$| - ^airflow-core/src/airflow/models/asset\.py$| - ^airflow-core/src/airflow/models/baseoperator\.py$| - ^airflow-core/src/airflow/models/callback\.py$| - ^airflow-core/src/airflow/models/connection\.py$| - ^airflow-core/src/airflow/models/dag\.py$| - ^airflow-core/src/airflow/models/dagrun\.py$| - ^airflow-core/src/airflow/models/deadline\.py$| - ^airflow-core/src/airflow/models/expandinput\.py$| - ^airflow-core/src/airflow/models/mappedoperator\.py$| - ^airflow-core/src/airflow/models/operator\.py$| - ^airflow-core/src/airflow/models/param\.py$| - ^airflow-core/src/airflow/models/renderedtifields\.py$| - ^airflow-core/src/airflow/models/serialized_dag\.py$| - ^airflow-core/src/airflow/models/taskinstance\.py$| - ^airflow-core/src/airflow/models/taskinstancekey\.py$| - ^airflow-core/src/airflow/models/taskmap\.py$| - ^airflow-core/src/airflow/models/taskmixin\.py$| - ^airflow-core/src/airflow/models/taskreschedule\.py$| - ^airflow-core/src/airflow/models/variable\.py$| - ^airflow-core/src/airflow/models/xcom\.py$| - ^airflow-core/src/airflow/models/xcom_arg\.py$| - ^airflow-core/src/airflow/operators/subdag\.py$| - ^airflow-core/src/airflow/plugins_manager\.py$| - ^airflow-core/src/airflow/providers_manager\.py$| - ^airflow-core/src/airflow/secrets/__init__.py$| - ^airflow-core/src/airflow/serialization/definitions/[_a-z]+\.py$| - ^airflow-core/src/airflow/serialization/enums\.py$| - ^airflow-core/src/airflow/serialization/helpers\.py$| - ^airflow-core/src/airflow/serialization/serialized_objects\.py$| - ^airflow-core/src/airflow/settings\.py$| - ^airflow-core/src/airflow/task/task_runner/bash_task_runner\.py$| - ^airflow-core/src/airflow/task/task_runner/standard_task_runner\.py$| - ^airflow-core/src/airflow/ti_deps/deps/mapped_task_upstream_dep\.py$| - ^airflow-core/src/airflow/ti_deps/deps/prev_dagrun_dep\.py$| - ^airflow-core/src/airflow/ti_deps/deps/trigger_rule_dep\.py$| - ^airflow-core/src/airflow/timetables/assets\.py$| - ^airflow-core/src/airflow/timetables/base\.py$| - ^airflow-core/src/airflow/timetables/simple\.py$| - ^airflow-core/src/airflow/utils/cli\.py$| - ^airflow-core/src/airflow/utils/context\.py$| - ^airflow-core/src/airflow/utils/dag_cycle_tester\.py$| - ^airflow-core/src/airflow/utils/dag_edges\.py$| - ^airflow-core/src/airflow/utils/dag_parsing_context\.py$| - ^airflow-core/src/airflow/utils/decorators\.py$| - ^airflow-core/src/airflow/utils/dot_renderer\.py$| - ^airflow-core/src/airflow/utils/edgemodifier\.py$| - ^airflow-core/src/airflow/utils/email\.py$| - ^airflow-core/src/airflow/utils/helpers\.py$| - ^airflow-core/src/airflow/utils/operator_helpers\.py$| - ^airflow-core/src/airflow/utils/session\.py$| - ^airflow-core/src/airflow/utils/task_group\.py$| - ^airflow-core/src/airflow/utils/trigger_rule\.py$| - ^airflow-core/src/airflow/utils/types\.py$ - ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - - id: check-schema-defaults - name: Check schema defaults match server-side defaults - entry: ./scripts/ci/prek/check_schema_defaults.py - language: python - files: ^airflow-core/src/airflow/serialization/schema\.json$|^airflow-core/src/airflow/serialization/serialized_objects\.py$ - pass_filenames: false + ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/.*\.py$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/(datamodels|routes|services|openapi)/.*\.py$ require_serial: true + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-contextmanager-class-decorators name: Check for problematic context manager class decorators entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml new file mode 100644 index 0000000000000..6adb2ba88edd6 --- /dev/null +++ b/airflow-core/.pre-commit-config.yaml @@ -0,0 +1,388 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.0.28' +repos: + - repo: local + hooks: + - id: check-taskinstance-tis-attrs + name: Check that TI and TIS have the same attributes + entry: ../scripts/ci/prek/check_ti_vs_tis_attributes.py + language: python + files: + (?x) + ^src/airflow/models/taskinstance\.py$| + ^src/airflow/models/taskinstancehistory\.py$ + pass_filenames: false + require_serial: true + - id: check-airflow-k8s-not-used + name: Check airflow.kubernetes imports are not used + language: python + files: ^src/airflow/.*\.py$ + require_serial: true + exclude: ^src/airflow/kubernetes/ + entry: ../scripts/ci/prek/check_airflow_imports.py + --pattern '^airflow\.kubernetes' + --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." + - id: check-common-compat-used-for-openlineage + name: Check common.compat is used for OL deprecated classes + language: python + files: ^src/airflow/.*\.py$ + require_serial: true + exclude: ^src/airflow/openlineage/.* + entry: ../scripts/ci/prek/check_airflow_imports.py + --pattern '^openlineage\.client\.(facet|run)' + --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." + - id: check-cncf-k8s-only-for-executors + name: Check cncf.kubernetes imports used for executors only + language: python + files: ^src/airflow/.*\.py$ + require_serial: true + exclude: > + (?x) + ^src/airflow/exceptions\.py$| + ^src/airflow/models/renderedtifields\.py$| + ^src/airflow/serialization/serialized_objects\.py$| + ^src/airflow/serialization/serializers/kubernetes\.py$| + ^src/airflow/utils/sqlalchemy\.py$ + entry: ../scripts/ci/prek/check_airflow_imports.py + --pattern '^airflow\.providers\.cncf\.kubernetes' + --message + "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." + - id: check-extra-packages-references + name: Checks setup extra packages + description: Checks if extras defined in hatch_build.py are listed in extra-packages-ref.rst file + language: python + files: ^docs/extra-packages-ref\.rst$|^hatch_build\.py$ + pass_filenames: false + entry: ../scripts/ci/prek/check_extra_packages_ref.py + - id: check-no-providers-in-core-examples + language: pygrep + name: No providers imports in core example DAGs + description: The core example DAGs have no dependencies other than standard provider or core Airflow + entry: "^\\s*from airflow\\.providers.(?!standard.)" + pass_filenames: true + files: ^src/airflow/example_dags/.*\.py$ + - id: check-only-new-session-with-provide-session + name: Check NEW_SESSION is only used with @provide_session + language: python + entry: ../scripts/ci/prek/new_session_in_provide_session.py + pass_filenames: true + files: ^src/airflow/.+\.py$ + exclude: ^src/airflow/serialization/pydantic/.* + - id: check-base-operator-partial-arguments + name: Check BaseOperator and partial() arguments + language: python + entry: ../scripts/ci/prek/check_base_operator_partial_arguments.py + pass_filenames: false + files: ^src/airflow/models/(?:base|mapped)operator\.py$ + - id: check-decorated-operator-implements-custom-name + name: Check @task decorator implements custom_operator_name + language: python + entry: ../scripts/ci/prek/decorator_operator_implements_custom_name.py + pass_filenames: true + files: ^src/airflow/.*\.py$ + - id: check-tests-in-the-right-folders + name: Check if tests are in the right folders + entry: ../scripts/ci/prek/check_tests_in_right_folders.py + language: python + files: ^tests/.*\.py$ + pass_filenames: true + require_serial: true + - id: lint-json-schema + name: Lint config_templates/config.yml + entry: ../scripts/ci/prek/lint_json_schema.py + args: + - --spec-file + - src/airflow/config_templates/config.yml.schema.json + language: python + pass_filenames: true + files: ^src/airflow/config_templates/config\.yml$ + require_serial: true + - id: check-code-deprecations + name: Check deprecations categories in decorators + entry: ../scripts/ci/prek/check_deprecations.py + language: python + pass_filenames: true + files: ^src/airflow/.*\.py$ + - id: check-significant-newsfragments-are-valid + name: Check significant newsfragments are valid + # Significant newsfragments follows a special format so that we can group information easily. + language: python + files: ^newsfragments/.*\.rst$ + entry: ../scripts/ci/prek/significant_newsfragments_checker.py + pass_filenames: false + # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes + # This is fast, so not too much downside + always_run: true + - id: create-missing-init-py-files-tests + name: Create missing init.py files in tests + entry: ../scripts/ci/prek/check_init_in_tests.py + language: python + pass_filenames: false + files: ^tests/.*\.py$ + - id: check-tests-unittest-testcase + name: Unit tests do not inherit from unittest.TestCase + description: Check that unit tests do not inherit from unittest.TestCase + entry: ../scripts/ci/prek/unittest_testcase.py + language: python + pass_filenames: true + files: ^tests/.*\.py$ + - id: check-no-fab-migrations + language: pygrep + name: Check no migration is done on FAB related table + description: > + FAB tables are no longer used in core Airflow but in FAB provider. + As such, it is forbidden to create migrations related to FAB tables in core Airflow. + Such migrations should be in FAB provider. To achieve this, a new capability must be implemented: + support migrations for providers. In other words, providers need to be able to specify migrations + so that, any FAB related migration (besides the legacy ones) is defined in FAB provider. + See https://github.com/apache/airflow/issues/32210 + entry: > + (?ix) + \bab_permission\b| + \bab_view_menu\b| + \bab_role\b| + \bab_permission_view\b| + \bab_permission_view_role\b| + \bab_user\b| + \bab_user_role\b| + \bab_register_user\b + pass_filenames: true + files: ^src/airflow/migrations/versions/.*\.py$ + exclude: + ^src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py$ + - id: ts-compile-lint-ui + name: Compile / format / lint UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^src/airflow/ui/.*\.(js|ts|tsx|yaml|css|json)$| + ^src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ + exclude: | + (?x) + ^src/airflow/ui/node-modules/.*| + ^src/airflow/ui/.pnpm-store + entry: ../scripts/ci/prek/ts_compile_lint_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true + - id: ts-compile-lint-simple-auth-manager-ui + name: Compile / format / lint simple auth manager UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^src/airflow/api_fastapi/auth/managers/simple/ui/.*\.(js|ts|tsx|yaml|css|json)$| + ^src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^src/airflow/api_fastapi/auth/managers/simple/openapi/.*\.yaml$ + exclude: | + (?x) + ^src/airflow/api_fastapi/node-modules/.*| + ^src/airflow/api_fastapi/.pnpm-store + entry: ../scripts/ci/prek/ts_compile_lint_simple_auth_manager_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built + - id: mypy-airflow-core + stages: ['pre-push'] + name: Run mypy for airflow-core + language: python + entry: ../scripts/ci/prek/mypy.py + files: ^.*\.py$ + require_serial: true + - id: mypy-airflow-core + stages: ['manual'] + name: Run mypy for airflow-core (manual) + language: python + entry: ../scripts/ci/prek/mypy_folder.py airflow-core + pass_filenames: false + files: ^.*\.py$ + require_serial: true + - id: generate-openapi-spec + name: Generate the FastAPI API spec + language: python + entry: ../scripts/ci/prek/generate_openapi_spec.py + pass_filenames: false + files: + (?x) + ^src/airflow/api_fastapi/.*\.py$| + ^src/airflow/api_fastapi/auth/managers/simple/.*\.py$ + exclude: ^src/airflow/api_fastapi/execution_api/.* + - id: check-i18n-json + name: Check i18n files validity + description: Check i18n files are valid json, have no TODOs, and auto-format them + language: python + files: ^src/airflow/ui/public/i18n/locales/.*\.json$ + entry: ../scripts/ci/prek/check_i18n_json.py + pass_filenames: false + - id: update-migration-references + name: Update migration ref doc + language: python + entry: ../scripts/ci/prek/migration_reference.py + pass_filenames: false + files: + (?x) + ^src/airflow/migrations/versions/.*\.py$| + ^docs/migrations-ref\.rst$ + - id: update-er-diagram + name: Update ER diagram + language: python + entry: ../scripts/ci/prek/update_er_diagram.py + pass_filenames: false + files: + (?x) + ^src/airflow/migrations/versions/.*\.py$| + ^docs/migrations-ref\.rst$ + - id: check-default-configuration + name: Check the default configuration + entry: ../scripts/ci/prek/check_default_configuration.py + language: python + require_serial: true + pass_filenames: false + files: ^src/airflow/config_templates/config\.yml$ + - id: check-airflow-version-checks-in-core + language: pygrep + name: No AIRFLOW_V_* imports in airflow-core + entry: "import AIRFLOW_V_" + files: ^.*\.py$ + pass_filenames: true + # TODO (@amoghrajesh): revisit last few in this list as they all rely on versioned + # secrets masker imports + exclude: > + (?x) + ^tests/integration/otel/dags/otel_test_dag_with_pause_between_tasks\.py$| + ^tests/integration/otel/dags/otel_test_dag_with_pause_in_task\.py$| + ^tests/integration/otel/test_otel\.py$| + ^tests/unit/core/test_configuration\.py$| + ^tests/unit/models/test_renderedtifields\.py$| + ^tests/unit/models/test_variable\.py$ + - id: check-sdk-imports + name: Check for SDK imports in core files + entry: ../scripts/ci/prek/check_sdk_imports.py + language: python + types: [python] + files: ^src/airflow/ + exclude: | + (?x) + # Allow SDK imports in these legitimate locations + ^src/airflow/example_dags/.*\.py$| + + # TODO: These files need to be refactored to remove SDK coupling + ^src/airflow/__init__\.py$| + ^src/airflow/api/common/mark_tasks\.py$| + ^src/airflow/api_fastapi/core_api/datamodels/assets\.py$| + ^src/airflow/api_fastapi/core_api/datamodels/connections\.py$| + ^src/airflow/api_fastapi/core_api/datamodels/hitl\.py$| + ^src/airflow/api_fastapi/core_api/datamodels/variables\.py$| + ^src/airflow/api_fastapi/core_api/routes/ui/grid\.py$| + ^src/airflow/api_fastapi/core_api/routes/ui/structure\.py$| + ^src/airflow/api_fastapi/core_api/services/public/connections\.py$| + ^src/airflow/api_fastapi/core_api/services/ui/connections\.py$| + ^src/airflow/api_fastapi/core_api/services/ui/grid\.py$| + ^src/airflow/api_fastapi/core_api/services/ui/task_group.py$| + ^src/airflow/api_fastapi/execution_api/routes/hitl\.py$| + ^src/airflow/api_fastapi/execution_api/routes/task_instances\.py$| + ^src/airflow/api_fastapi/logging/decorators\.py$| + ^src/airflow/assets/evaluation\.py$| + ^src/airflow/assets/manager\.py$| + ^src/airflow/cli/commands/connection_command\.py$| + ^src/airflow/cli/commands/task_command\.py$| + ^src/airflow/cli/commands/triggerer_command.py$| + ^src/airflow/configuration\.py$| + ^src/airflow/dag_processing/collection\.py$| + ^src/airflow/dag_processing/manager\.py$| + ^src/airflow/dag_processing/processor\.py$| + ^src/airflow/dag_processing/dagbag\.py$| + ^src/airflow/datasets/metadata\.py$| + ^src/airflow/exceptions\.py$| + ^src/airflow/executors/local_executor\.py$| + ^src/airflow/jobs/triggerer_job_runner\.py$| + ^src/airflow/lineage/hook\.py$| + ^src/airflow/listeners/spec/asset\.py$| + ^src/airflow/listeners/spec/taskinstance\.py$| + ^src/airflow/logging/remote\.py$| + ^src/airflow/models/__init__\.py$| + ^src/airflow/models/asset\.py$| + ^src/airflow/models/baseoperator\.py$| + ^src/airflow/models/callback\.py$| + ^src/airflow/models/connection\.py$| + ^src/airflow/models/dag\.py$| + ^src/airflow/models/dagrun\.py$| + ^src/airflow/models/deadline\.py$| + ^src/airflow/models/expandinput\.py$| + ^src/airflow/models/mappedoperator\.py$| + ^src/airflow/models/operator\.py$| + ^src/airflow/models/param\.py$| + ^src/airflow/models/renderedtifields\.py$| + ^src/airflow/models/serialized_dag\.py$| + ^src/airflow/models/taskinstance\.py$| + ^src/airflow/models/taskinstancekey\.py$| + ^src/airflow/models/taskmap\.py$| + ^src/airflow/models/taskmixin\.py$| + ^src/airflow/models/taskreschedule\.py$| + ^src/airflow/models/variable\.py$| + ^src/airflow/models/xcom\.py$| + ^src/airflow/models/xcom_arg\.py$| + ^src/airflow/operators/subdag\.py$| + ^src/airflow/plugins_manager\.py$| + ^src/airflow/providers_manager\.py$| + ^src/airflow/secrets/__init__.py$| + ^src/airflow/serialization/definitions/[_a-z]+\.py$| + ^src/airflow/serialization/enums\.py$| + ^src/airflow/serialization/helpers\.py$| + ^src/airflow/serialization/serialized_objects\.py$| + ^src/airflow/settings\.py$| + ^src/airflow/task/task_runner/bash_task_runner\.py$| + ^src/airflow/task/task_runner/standard_task_runner\.py$| + ^src/airflow/ti_deps/deps/mapped_task_upstream_dep\.py$| + ^src/airflow/ti_deps/deps/prev_dagrun_dep\.py$| + ^src/airflow/ti_deps/deps/trigger_rule_dep\.py$| + ^src/airflow/timetables/assets\.py$| + ^src/airflow/timetables/base\.py$| + ^src/airflow/timetables/simple\.py$| + ^src/airflow/utils/cli\.py$| + ^src/airflow/utils/context\.py$| + ^src/airflow/utils/dag_cycle_tester\.py$| + ^src/airflow/utils/dag_edges\.py$| + ^src/airflow/utils/dag_parsing_context\.py$| + ^src/airflow/utils/decorators\.py$| + ^src/airflow/utils/dot_renderer\.py$| + ^src/airflow/utils/edgemodifier\.py$| + ^src/airflow/utils/email\.py$| + ^src/airflow/utils/helpers\.py$| + ^src/airflow/utils/operator_helpers\.py$| + ^src/airflow/utils/session\.py$| + ^src/airflow/utils/task_group\.py$| + ^src/airflow/utils/trigger_rule\.py$| + ^src/airflow/utils/types\.py$ + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE + - id: check-schema-defaults + name: Check schema defaults match server-side defaults + entry: ../scripts/ci/prek/check_schema_defaults.py + language: python + files: + (?x) + ^src/airflow/serialization/schema\.json$| + ^src/airflow/serialization/serialized_objects\.py$ + pass_filenames: false + require_serial: true diff --git a/scripts/ci/prek/check_tests_in_right_folders.py b/scripts/ci/prek/check_tests_in_right_folders.py index 7c9f34d182d00..5737594c68414 100755 --- a/scripts/ci/prek/check_tests_in_right_folders.py +++ b/scripts/ci/prek/check_tests_in_right_folders.py @@ -83,20 +83,16 @@ ] EXCEPTIONS = [ - "airflow-core/tests/__init__.py", - "airflow-core/tests/unit/__init__.py", - "airflow-core/tests/system/__init__.py", - "airflow-core/tests/system/conftest.py", - "airflow-core/tests/system/example_empty.py", - "airflow-core/tests/integration/__init__.py", - "airflow-core/tests/conftest.py", + "tests/system/conftest.py", + "tests/system/example_empty.py", + "tests/conftest.py", ] if __name__ == "__main__": files = sys.argv[1:] - MATCH_TOP_LEVEL_TEST_FILES = re.compile(r"airflow-core/tests/unit/[^/]+\.py") - files = [file for file in files if file not in EXCEPTIONS] + MATCH_TOP_LEVEL_TEST_FILES = re.compile(r"tests/unit/[^/]+\.py") + files = [file for file in files if file not in EXCEPTIONS and not file.endswith("/__init__.py")] errors = False top_level_files = [file for file in files if MATCH_TOP_LEVEL_TEST_FILES.match(file)] @@ -108,9 +104,9 @@ errors = True for file in files: if not any( - file.startswith(f"airflow-core/tests/unit/{folder}/") - or file.startswith(f"airflow-core/tests/integration/{folder}/") - or file.startswith(f"airflow-core/tests/system/{folder}/") + file.startswith(f"tests/unit/{folder}/") + or file.startswith(f"tests/integration/{folder}/") + or file.startswith(f"tests/system/{folder}/") for folder in POSSIBLE_TEST_FOLDERS ): console.print(