From e36bc2642dc6f5b4fe462d17370d9e4514b614ef Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 14 Jun 2025 10:38:40 +0200 Subject: [PATCH] Optimize ui lint pre-commits Running pre-commits for typescript compilation, linting formatting is currently very slow. The pre-commit that is doing it took more than 2.5 minutes - always, independently which files changed. Also, no matter whether UI or simple auth manager UI changed, both were compiled. This is the first iteration of speeding up the pre-commit especially for incremental changes. The following changes were implemented: * the scripts were split into ui and simple-auth-manager-ui * timing information and verbose commands displayed so that we can measure which commands take how much time * the pre-commmit now passes all the files as input (and pre-commit is serialized) - which means that some commands can only be run on the subset of files that changed * tsc step is modified to only run on a subset of .ts and .tsx files that were changed (via dynamic extension of the project) Overall when just few files change, the pre-commit should now complete in less than 10-20 seconds Still some optimisations are possible in the future. --- .pre-commit-config.yaml | 48 ++++++++--- contributing-docs/08_static_code_checks.rst | 4 +- .../doc/images/output_static-checks.svg | 86 ++++++++++--------- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 3 +- .../airflow_breeze/utils/selective_checks.py | 3 +- dev/breeze/tests/test_selective_checks.py | 21 +++-- .../ci/pre_commit/common_precommit_utils.py | 54 ++++++++++++ scripts/ci/pre_commit/compile_lint_ui.py | 51 ----------- .../ts_compile_lint_simple_auth_manager_ui.py | 61 +++++++++++++ scripts/ci/pre_commit/ts_compile_lint_ui.py | 61 +++++++++++++ 11 files changed, 276 insertions(+), 118 deletions(-) delete mode 100755 scripts/ci/pre_commit/compile_lint_ui.py create mode 100755 scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py create mode 100755 scripts/ci/pre_commit/ts_compile_lint_ui.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2f22b1d9e410c..a5b572436f32c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1283,6 +1283,40 @@ repos: files: ^go-sdk/ types: [go] language: golang + - id: ts-compile-lint-ui + name: Compile / format / lint UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^airflow-core/src/airflow/ui/.*\.(js|ts|tsx|yaml|css|json)| + ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ + exclude: | + (?x) + ^airflow-core/src/airflow/ui/node-modules/.*| + ^airflow-core/src/airflow/ui/.pnpm-store + entry: ./scripts/ci/pre_commit/ts_compile_lint_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true + - id: ts-compile-lint-simple-auth-manager-ui + name: Compile / format / lint simple auth manager UI + description: TS types generation / ESLint / Prettier new UI files + language: node + files: | + (?x) + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/.*\.(js|ts|tsx|yaml|css|json)| + ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ + exclude: | + (?x) + ^airflow-core/src/airflow/api_fastapi/node-modules/.*| + ^airflow-core/src/airflow/api_fastapi/.pnpm-store + entry: ./scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py + additional_dependencies: ['pnpm@9.7.1'] + pass_filenames: true + require_serial: true ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: mypy-dev @@ -1406,20 +1440,6 @@ repos: entry: ./scripts/ci/pre_commit/check_i18n_json.py additional_dependencies: ['rich>=12.4.4'] pass_filenames: false - - id: ts-compile-format-lint-ui - name: Compile / format / lint UI - description: TS types generation / ESLint / Prettier new UI files - language: node - types_or: [javascript, ts, tsx, yaml, css, json] - files: | - (?x) - ^airflow-core/src/airflow/ui/| - ^airflow-core/src/airflow/api_fastapi/core_api/openapi/.*\.yaml$| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/openapi/v1.*\.yaml$ - entry: ./scripts/ci/pre_commit/compile_lint_ui.py - additional_dependencies: ['pnpm@9.7.1'] - pass_filenames: false - id: check-provider-yaml-valid name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index bc71c54ff0ece..21e4da200bc3a 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -366,7 +366,9 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | trailing-whitespace | Remove trailing whitespace at end of line | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-format-lint-ui | Compile / format / lint UI | * | +| ts-compile-lint-simple-auth-manager-ui | Compile / format / lint simple auth manager UI | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ +| ts-compile-lint-ui | Compile / format / lint UI | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-black-version | Update black versions everywhere (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index 72dd6b4dfc65f..4aac075f007ad 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -1,4 +1,4 @@ - +