From 5d2b378527a3fd1e1efd18096137c80d6fadee7a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 14 Jun 2025 15:25:01 +0200 Subject: [PATCH] [v3-0-test] Optimize ui lint pre-commits (#51725) Running pre-commits for typescript compilation, linting formatting is currently very slow. The pre-commit that is doing it took more than 2.5 minutes - always, independently which files changed. Also, no matter whether UI or simple auth manager UI changed, both were compiled. This is the first iteration of speeding up the pre-commit especially for incremental changes. The following changes were implemented: * the scripts were split into ui and simple-auth-manager-ui * timing information and verbose commands displayed so that we can measure which commands take how much time * the pre-commmit now passes all the files as input (and pre-commit is serialized) - which means that some commands can only be run on the subset of files that changed * tsc step is modified to only run on a subset of .ts and .tsx files that were changed (via dynamic extension of the project) Overall when just few files change, the pre-commit should now complete in less than 10-20 seconds Still some optimisations are possible in the future. (cherry picked from commit 52be0dd4c471101e1afe342b8ef8d716bde2525c) Co-authored-by: Jarek Potiuk --- .pre-commit-config.yaml | 73 ++++++++++++---- contributing-docs/08_static_code_checks.rst | 4 +- .../doc/images/output_static-checks.svg | 86 ++++++++++--------- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 3 +- .../airflow_breeze/utils/selective_checks.py | 3 +- dev/breeze/tests/test_selective_checks.py | 21 +++-- .../ci/pre_commit/common_precommit_utils.py | 54 ++++++++++++ scripts/ci/pre_commit/compile_lint_ui.py | 51 ----------- .../ts_compile_lint_simple_auth_manager_ui.py | 61 +++++++++++++ scripts/ci/pre_commit/ts_compile_lint_ui.py | 61 +++++++++++++ 11 files changed, 296 insertions(+), 123 deletions(-) delete mode 100755 scripts/ci/pre_commit/compile_lint_ui.py create mode 100755 scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py create mode 100755 scripts/ci/pre_commit/ts_compile_lint_ui.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e5c88598f172a..fe166b32a39ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,7 @@ default_stages: [pre-commit, pre-push] default_language_version: python: python3 node: 22.16.0 + golang: 1.24.0 minimum_pre_commit_version: '3.2.0' exclude: ^.*/.*_vendor/ repos: @@ -301,6 +302,7 @@ repos: ^.*airflow\.template\.yaml$| ^.*init_git_sync\.template\.yaml$| ^chart/(?:templates|files)/.*\.yaml$| + ^helm-tests/tests/chart_utils/keda.sh_scaledobjects\.yaml$| .*/v1.*\.yaml$| ^.*openapi.*\.yaml$| ^\.pre-commit-config\.yaml$| @@ -328,7 +330,17 @@ repos: The word(s) should be in lowercase." && exec codespell "$@"' -- language: python types: [text] - exclude: material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$|.*/dist/.* + exclude: > + (?x) + material-icons\.css$| + ^images/.*$| + ^RELEASE_NOTES\.txt$| + ^.*package-lock\.json$| + ^.*/kinglear\.txt$| + ^.*pnpm-lock\.yaml$| + .*/dist/.*| + ^airflow-core/src/airflow/ui/src/i18n/locales/de/| + ^airflow-core/src/airflow/ui/src/i18n/locales/pl/ args: - --ignore-words=docs/spelling_wordlist.txt - --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md,*.svg @@ -390,7 +402,7 @@ repos: types_or: [python, pyi] args: [--fix] require_serial: true - additional_dependencies: ['ruff==0.11.2'] + additional_dependencies: ['ruff==0.11.13'] exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$ - id: ruff-format name: Run 'ruff format' @@ -400,14 +412,14 @@ repos: types_or: [python, pyi] args: [] require_serial: true - additional_dependencies: ['ruff==0.11.2'] + additional_dependencies: ['ruff==0.11.13'] exclude: ^airflow-core/tests/unit/dags/test_imports\.py$ - id: replace-bad-characters name: Replace bad characters entry: ./scripts/ci/pre_commit/replace_bad_characters.py language: python types: [file, text] - exclude: ^clients/gen/go\.sh$|^\.gitmodules$|.*/dist/.* + exclude: ^clients/gen/go\.sh$|^\.gitmodules$|.*/dist/.*|\.go$|/go\.(mod|sum)$ additional_dependencies: ['rich>=12.4.4'] - id: lint-dockerfile name: Lint Dockerfile @@ -606,7 +618,9 @@ repos: pass_filenames: true exclude: > (?x) + ^airflow-core/src/airflow/ui/src/i18n/config\.ts$| ^airflow-core/src/airflow/ui/openapi-gen/| + ^airflow-core/src/airflow/ui/src/i18n/locales/de/README\.md$| ^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$| ^airflow-core/src/airflow/config_templates/| ^airflow-core/src/airflow/models/baseoperator\.py$| @@ -628,6 +642,7 @@ repos: ^providers/google/src/airflow/providers/google/cloud/operators/cloud_build\.py$| ^providers/google/src/airflow/providers/google/cloud/operators/dataproc\.py$| ^providers/google/src/airflow/providers/google/cloud/operators/mlengine\.py$| + ^providers/keycloak/src/airflow/providers/keycloak/auth_manager/cli/definition.py| ^providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/cosmos\.py$| ^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/hooks/winrm\.py$| ^airflow-core/docs/.*commits\.rst$| @@ -641,6 +656,7 @@ repos: ^airflow-core/src/airflow/utils/trigger_rule\.py$| ^chart/values.schema\.json$| ^helm-tests/tests/chart_utils/helm_template_generator\.py$| + ^helm-tests/tests/chart_utils/ingress-networking-v1beta1\.json$| ^dev/| ^devel-common/src/docs/README\.rst$| ^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$| @@ -856,7 +872,6 @@ repos: - id: compile-fab-assets name: Compile FAB provider assets language: node - 'types_or': [javascript, ts, tsx] files: ^providers/fab/.*/www/ entry: ./scripts/ci/pre_commit/compile_fab_assets.py pass_filenames: false @@ -1231,6 +1246,40 @@ repos: files: ^airflow-core/src/airflow/migrations/versions/.*\.py$ exclude: airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py + - id: ts-compile-lint-ui + name: Compile / format / lint UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^airflow-core/src/airflow/ui/.*\.(js|ts|tsx|yaml|css|json)| + ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ + exclude: | + (?x) + ^airflow-core/src/airflow/ui/node-modules/.*| + ^airflow-core/src/airflow/ui/.pnpm-store + entry: ./scripts/ci/pre_commit/ts_compile_lint_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true + - id: ts-compile-lint-simple-auth-manager-ui + name: Compile / format / lint simple auth manager UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/.*\.(js|ts|tsx|yaml|css|json)| + ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ + exclude: | + (?x) + ^airflow-core/src/airflow/api_fastapi/node-modules/.*| + ^airflow-core/src/airflow/api_fastapi/.pnpm-store + entry: ./scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: mypy-dev @@ -1346,20 +1395,6 @@ repos: files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* additional_dependencies: ['rich>=12.4.4', 'openapi-spec-validator>=0.7.1'] - - id: ts-compile-format-lint-ui - name: Compile / format / lint UI - description: TS types generation / ESLint / Prettier new UI files - language: node - types_or: [javascript, ts, tsx, yaml, css, json] - files: | - (?x) - ^airflow-core/src/airflow/ui/| - ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ - entry: ./scripts/ci/pre_commit/compile_lint_ui.py - additional_dependencies: ['pnpm@9.7.1'] - pass_filenames: false - id: check-provider-yaml-valid name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 9e4a4fbd5a576..ab4ec43edec23 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -352,7 +352,9 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | trailing-whitespace | Remove trailing whitespace at end of line | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-format-lint-ui | Compile / format / lint UI | * | +| ts-compile-lint-simple-auth-manager-ui | Compile / format / lint simple auth manager UI | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ +| ts-compile-lint-ui | Compile / format / lint UI | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-black-version | Update black versions everywhere (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index 4dd66c187ffda..8ff92c29da3f7 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -1,4 +1,4 @@ - +