Enable dynamic generation of assets for Edge UI Plugin

.github/workflows/generate-constraints.yml

@@ -78,6 +78,13 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           persist-credentials: false
+      - name: "Install prek"
+        uses: ./.github/actions/install-prek
+        id: prek
+        with:
+          python-version: ${{ matrix.python-version }}
+          platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:

.github/workflows/test-providers.yml

@@ -95,6 +95,13 @@ jobs:
       - name: "Free up disk space"
         shell: bash
         run: ./scripts/tools/free_up_disk_space.sh
+      - name: "Install prek"
+        uses: ./.github/actions/install-prek
+        id: prek
+        with:
+          python-version: ${{ inputs.default-python-version }}
+          platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
@@ -197,6 +204,13 @@ jobs:
       - name: "Free up disk space"
         shell: bash
         run: ./scripts/tools/free_up_disk_space.sh
+      - name: "Install prek"
+        uses: ./.github/actions/install-prek
+        id: prek
+        with:
+          python-version: ${{ matrix.compat.python-version }}
+          platform: ${{ inputs.platform }}
+          save-cache: false
       - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:

airflow-core/src/airflow/provider.yaml.schema.json

@@ -31,6 +31,15 @@
                 "removed"
             ]
         },
+        "build-system": {
+            "description": "Build-system of provider: Defaults to flit_core but also can be hatchling.",
+            "type": "string",
+            "default": "flit_core",
+            "enum": [
+                "flit_core",
+                "hatchling"
+            ]
+        },
         "excluded-python-versions": {
             "description": "List of python versions excluded for that provider",
             "type": "array",

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -120,6 +120,7 @@
 from airflow_breeze.utils.packages import expand_all_provider_distributions
 from airflow_breeze.utils.path_utils import (
     AIRFLOW_ROOT_PATH,
+    EDGE_PLUGIN_PREK_HOOK,
     cleanup_python_generated_files,
 )
 from airflow_breeze.utils.platforms import get_normalized_platform
@@ -654,8 +655,12 @@ def start_airflow(
     perform_environment_checks(quiet=False)
     if use_airflow_version is None and not skip_assets_compilation:
         assert_prek_installed()
+        # Compile edge assets as well if needed
+        additional_assets = [EDGE_PLUGIN_PREK_HOOK] if executor and "EdgeExecutor" in executor else []
         # Now with the /ui project, lets only do a static build of /www and focus on the /ui
-        run_compile_ui_assets(dev=dev_mode, run_in_background=True, force_clean=False)
+        run_compile_ui_assets(
+            dev=dev_mode, run_in_background=True, force_clean=False, additional_ui_hooks=additional_assets
+        )
     airflow_constraints_reference = _determine_constraint_branch_used(
         airflow_constraints_reference, use_airflow_version
     )

dev/breeze/src/airflow_breeze/commands/ui_commands.py

@@ -711,7 +711,7 @@ def compile_ui_assets(dev: bool, force_clean: bool):
     perform_environment_checks()
     assert_prek_installed()
     compile_ui_assets_result = run_compile_ui_assets(
-        dev=dev, run_in_background=False, force_clean=force_clean
+        dev=dev, run_in_background=False, force_clean=force_clean, additional_ui_hooks=[]
     )
     if compile_ui_assets_result.returncode != 0:
         get_console().print("[warn]New assets were generated[/]")

dev/breeze/src/airflow_breeze/prepare_providers/provider_distributions.py

@@ -29,9 +29,11 @@
     get_latest_provider_tag,
     get_not_ready_provider_ids,
     get_provider_details,
+    get_provider_distributions_metadata,
     get_removed_provider_ids,
     tag_exists_for_provider,
 )
+from airflow_breeze.utils.path_utils import AIRFLOW_DIST_PATH
 from airflow_breeze.utils.run_utils import run_command
 from airflow_breeze.utils.version_utils import is_local_package_version
 
@@ -97,37 +99,64 @@ def build_provider_distribution(
         f"\n[info]Building provider package: {provider_id} "
         f"in format {distribution_format} in {target_provider_root_sources_path}\n"
     )
-    command: list[str] = [sys.executable, "-m", "flit", "build", "--no-setup-py", "--use-vcs"]
-    get_console().print(
-        "[warning]Workaround wheel-only package bug in flit by building both and removing sdist."
-    )
-    # Workaround https://github.com/pypa/flit/issues/743 bug in flit that causes .gitignored files
-    # to be included in the package when --format wheel is used
-    remove_sdist = False
-    if distribution_format == "wheel":
-        distribution_format = "both"
-        remove_sdist = True
-    if distribution_format != "both":
-        command.extend(["--format", distribution_format])
-    try:
-        run_command(
-            command,
-            check=True,
-            cwd=target_provider_root_sources_path,
-            env={
-                "SOURCE_DATE_EPOCH": str(get_provider_details(provider_id).source_date_epoch),
-            },
+    provider_info = get_provider_distributions_metadata().get(provider_id)
+    if not provider_info:
+        raise RuntimeError(f"The provider {provider_id} has no provider.yaml defined.")
+    build_backend = provider_info.get("build-system", "flit_core")
+    if build_backend == "flit_core":
+        command: list[str] = [sys.executable, "-m", "flit", "build", "--no-setup-py", "--use-vcs"]
+        get_console().print(
+            "[warning]Workaround wheel-only package bug in flit by building both and removing sdist."
         )
-    except subprocess.CalledProcessError as ex:
-        get_console().print("[error]The command returned an error %s", ex)
+        # Workaround https://github.com/pypa/flit/issues/743 bug in flit that causes .gitignored files
+        # to be included in the package when --format wheel is used
+        remove_sdist = False
+        if distribution_format == "wheel":
+            distribution_format = "both"
+            remove_sdist = True
+        if distribution_format != "both":
+            command.extend(["--format", distribution_format])
+        try:
+            run_command(
+                command,
+                check=True,
+                cwd=target_provider_root_sources_path,
+                env={
+                    "SOURCE_DATE_EPOCH": str(get_provider_details(provider_id).source_date_epoch),
+                },
+            )
+        except subprocess.CalledProcessError as ex:
+            get_console().print(f"[error]The command returned an error {ex}")
+            raise PrepareReleasePackageErrorBuildingPackageException()
+        if remove_sdist:
+            get_console().print("[warning]Removing sdist file to workaround flit bug on wheel-only packages")
+            # Remove the sdist file if it was created
+            package_prefix = "apache_airflow_providers_" + provider_id.replace(".", "_")
+            for file in (target_provider_root_sources_path / "dist").glob(f"{package_prefix}*.tar.gz"):
+                get_console().print(f"[info]Removing {file} to workaround flit bug on wheel-only packages")
+                file.unlink(missing_ok=True)
+    elif build_backend == "hatchling":
+        command = [sys.executable, "-m", "hatch", "build", "-c", "-t", "custom"]
+        if distribution_format == "sdist" or distribution_format == "both":
+            command += ["-t", "sdist"]
+        if distribution_format == "wheel" or distribution_format == "both":
+            command += ["-t", "wheel"]
+        try:
+            run_command(
+                cmd=command,
+                cwd=target_provider_root_sources_path,
+                env={
+                    "SOURCE_DATE_EPOCH": str(get_provider_details(provider_id).source_date_epoch),
+                },
+                check=True,
+            )
+        except subprocess.CalledProcessError as ex:
+            get_console().print(f"[error]The command returned an error {ex}")
+            raise PrepareReleasePackageErrorBuildingPackageException()
+        shutil.copytree(target_provider_root_sources_path / "dist", AIRFLOW_DIST_PATH, dirs_exist_ok=True)
+    else:
+        get_console().print(f"[error]Unknown/unsupported build backend {build_backend}")
         raise PrepareReleasePackageErrorBuildingPackageException()
-    if remove_sdist:
-        get_console().print("[warning]Removing sdist file to workaround flit bug on wheel-only packages")
-        # Remove the sdist file if it was created
-        package_prefix = "apache_airflow_providers_" + provider_id.replace(".", "_")
-        for file in (target_provider_root_sources_path / "dist").glob(f"{package_prefix}*.tar.gz"):
-            get_console().print(f"[info]Removing {file} to workaround flit bug on wheel-only packages")
-            file.unlink(missing_ok=True)
     get_console().print(
         f"\n[info]Prepared provider package {provider_id} in format {distribution_format}[/]\n"
     )

dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2

@@ -39,8 +39,17 @@
 # IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
 # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
 [build-system]
+{% if BUILD_SYSTEM == "flit_core" -%}
 requires = ["flit_core==3.12.0"]
 build-backend = "flit_core.buildapi"
+{% else -%}
+requires = [
+{% for requires in HATCH_REQUIRES %}
+    "{{ requires }}",
+{% endfor -%}
+]
+build-backend = "hatchling.build"
+{% endif %}
 
 [project]
 name = "{{ PACKAGE_PIP_NAME }}"
@@ -143,5 +152,45 @@ provider_info = "airflow.providers.{{ PROVIDER_ID }}.get_provider_info:get_provi
 {% endfor %}
 {% endif %}
 
+{% if BUILD_SYSTEM == "flit_core" -%}
 [tool.flit.module]
 name = "airflow.providers.{{ PROVIDER_ID }}"
+{% else -%}
+[tool.hatch.version]
+path = "src/airflow/providers/{{ PROVIDER_ID }}/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "docs",
+    "src/airflow/providers/{{ PROVIDER_ID }}",
+    "tests",
+    "NOTICE"
+]
+exclude = [
+{% for exclude in HATCH_EXCLUDES %}
+    "{{ exclude }}",
+{% endfor -%}
+]
+
+[tool.hatch.build.targets.custom]
+path = "./hatch_build.py"
+
+artifacts = [
+{% for artifact in HATCH_ARTIFACTS %}
+    "{{ artifact }}",
+{% endfor -%}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ['src/airflow']
+artifacts = [
+{% for artifact in HATCH_ARTIFACTS %}
+    "{{ artifact }}",
+{% endfor -%}
+]
+exclude = [
+{% for exclude in HATCH_EXCLUDES %}
+    "{{ exclude }}",
+{% endfor -%}
+]
+{% endif -%}

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -43,6 +43,7 @@
 from airflow_breeze.utils.path_utils import (
     AIRFLOW_ORIGINAL_PROVIDERS_DIR,
     AIRFLOW_PROVIDERS_ROOT_PATH,
+    AIRFLOW_PYPROJECT_TOML_FILE_PATH,
     BREEZE_SOURCES_PATH,
     DOCS_ROOT,
     PREVIOUS_AIRFLOW_PROVIDERS_NS_PACKAGE_PATH,
@@ -92,6 +93,10 @@ class ProviderPackageDetails(NamedTuple):
     plugins: list[PluginInfo]
     removed: bool
     extra_project_metadata: str | None = None
+    build_system: str | None = None
+    hatch_artifacts: list[str] = []
+    hatch_excludes: list[str] = []
+    hatch_requires: list[str] = []
 
 
 class PackageSuspendedException(Exception):
@@ -553,6 +558,10 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails:
     provider_yaml_path = get_provider_yaml(provider_id)
     pyproject_toml = load_pyproject_toml(provider_yaml_path.parent / "pyproject.toml")
     dependencies = pyproject_toml["project"]["dependencies"]
+    hatch_targets: dict = pyproject_toml.get("tool", {}).get("hatch", {}).get("build", {}).get("targets", {})
+    hatch_artifacts = hatch_targets.get("custom", {}).get("artifacts", [])
+    hatch_excludes = hatch_targets.get("sdist", {}).get("exclude", [])
+    core_pyproject_toml = load_pyproject_toml(AIRFLOW_PYPROJECT_TOML_FILE_PATH)
     changelog_path = provider_yaml_path.parent / "docs" / "changelog.rst"
     documentation_provider_distribution_path = get_documentation_package_path(provider_id)
     root_provider_path = provider_yaml_path.parent
@@ -577,6 +586,10 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails:
         plugins=plugins,
         removed=provider_info["state"] == "removed",
         extra_project_metadata=provider_info.get("extra-project-metadata", ""),
+        build_system=provider_info.get("build-system", "flit_core"),
+        hatch_artifacts=hatch_artifacts,
+        hatch_excludes=hatch_excludes,
+        hatch_requires=core_pyproject_toml.get("build-system", {}).get("requires", []),
     )
 
 
@@ -950,6 +963,12 @@ def regenerate_pyproject_toml(
         formatted_cross_provider_dependencies = ""
     context["CROSS_PROVIDER_DEPENDENCIES"] = formatted_cross_provider_dependencies
     context["DEPENDENCY_GROUPS"] = formatted_dependency_groups
+    context["BUILD_SYSTEM"] = provider_details.build_system
+    if context["BUILD_SYSTEM"] == "hatchling":
+        context["HATCH_ARTIFACTS"] = provider_details.hatch_artifacts
+        context["HATCH_EXCLUDES"] = provider_details.hatch_excludes
+        context["HATCH_REQUIRES"] = provider_details.hatch_requires
+
     pyproject_toml_content = render_template(
         template_name="pyproject",
         context=context,

dev/breeze/src/airflow_breeze/utils/path_utils.py

@@ -292,6 +292,12 @@ def find_airflow_root_path_to_operate_on() -> Path:
 FAST_API_SIMPLE_AUTH_MANAGER_VITE_MANIFEST_PATH = (
     FAST_API_SIMPLE_AUTH_MANAGER_DIST_PATH / ".vite" / "manifest.json"
 )
+EDGE_PLUGIN_PATH = (
+    AIRFLOW_PROVIDERS_ROOT_PATH / "edge3" / "src" / "airflow" / "providers" / "edge3" / "plugins" / "www"
+)
+EDGE_PLUGIN_UI_NODE_MODULES_PATH = EDGE_PLUGIN_PATH / "node_modules"
+EDGE_PLUGIN_UI_DIST_PATH = EDGE_PLUGIN_PATH / "dist"
+EDGE_PLUGIN_PREK_HOOK = "compile-edge-assets"
 
 DAGS_PATH = AIRFLOW_ROOT_PATH / "dags"
 FILES_PATH = AIRFLOW_ROOT_PATH / "files"

dev/breeze/src/airflow_breeze/utils/run_utils.py

@@ -40,6 +40,11 @@
 from airflow_breeze.utils.functools_cache import clearable_cache
 from airflow_breeze.utils.path_utils import (
     AIRFLOW_ROOT_PATH,
+    EDGE_PLUGIN_PREK_HOOK,
+    EDGE_PLUGIN_UI_DIST_PATH,
+    EDGE_PLUGIN_UI_NODE_MODULES_PATH,
+    FAST_API_SIMPLE_AUTH_MANAGER_DIST_PATH,
+    FAST_API_SIMPLE_AUTH_MANAGER_NODE_MODULES_PATH,
     UI_ASSET_COMPILE_LOCK,
     UI_ASSET_HASH_PATH,
     UI_ASSET_OUT_DEV_MODE_FILE,
@@ -496,21 +501,27 @@ def kill_process_group(gid: int):
         os.killpg(gid, signal.SIGTERM)
 
 
-def clean_ui_assets():
+def _clean_ui_assets(additional_ui_hooks: list[str]):
     get_console().print("[info]Cleaning ui assets[/]")
     UI_ASSET_HASH_PATH.unlink(missing_ok=True)
     shutil.rmtree(UI_NODE_MODULES_PATH, ignore_errors=True)
     shutil.rmtree(UI_DIST_PATH, ignore_errors=True)
+    shutil.rmtree(FAST_API_SIMPLE_AUTH_MANAGER_NODE_MODULES_PATH, ignore_errors=True)
+    shutil.rmtree(FAST_API_SIMPLE_AUTH_MANAGER_DIST_PATH, ignore_errors=True)
+    if EDGE_PLUGIN_PREK_HOOK in additional_ui_hooks:
+        shutil.rmtree(EDGE_PLUGIN_UI_NODE_MODULES_PATH, ignore_errors=True)
+        shutil.rmtree(EDGE_PLUGIN_UI_DIST_PATH, ignore_errors=True)
     get_console().print("[success]Cleaned ui assets[/]")
 
 
 def run_compile_ui_assets(
     dev: bool,
     run_in_background: bool,
     force_clean: bool,
+    additional_ui_hooks: list[str],
 ):
     if force_clean:
-        clean_ui_assets()
+        _clean_ui_assets(additional_ui_hooks)
     if dev:
         get_console().print("\n[warning] The command below will run forever until you press Ctrl-C[/]\n")
         get_console().print(
@@ -524,6 +535,7 @@ def run_compile_ui_assets(
         "--hook-stage",
         "manual",
         "compile-ui-assets-dev" if dev else "compile-ui-assets",
+        *additional_ui_hooks,
         "--all-files",
         "--verbose",
     ]

providers/edge3/.gitignore

@@ -0,0 +1,2 @@
+# Do not commit hash files, this is just to speed-up local builds
+www-hash.txt

providers/edge3/.pre-commit-config.yaml

@@ -49,7 +49,8 @@ repos:
       - id: compile-edge-assets
         name: Compile Edge provider assets
         language: node
+        stages: ['pre-commit', 'manual']
         files: ^src/airflow/providers/edge3/plugins/www
         entry: ../../scripts/ci/prek/compile_provider_assets.py edge
         pass_filenames: false
-        additional_dependencies: ['yarn@1.22.21']
+        additional_dependencies: ['pnpm@10.25.0']