[v3-1-test] Implement integration test for airflowctl with 3.1 (#56124)

.dockerignore

@@ -47,6 +47,7 @@
 !helm-tests
 !kubernetes-tests
 !task-sdk-tests
+!airflow-ctl-tests
 !shared/
 
 # Add scripts so that we can use them inside the container

.github/workflows/additional-prod-image-tests.yml

@@ -191,3 +191,34 @@ jobs:
         id: breeze
       - name: "Run Task SDK integration tests"
         run: breeze testing task-sdk-integration-tests
+
+  airflow-ctl-integration-tests:
+    timeout-minutes: 60
+    name: "Airflow CTL integration tests with PROD image"
+    runs-on: ${{ fromJSON(inputs.runners) }}
+    env:
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_USERNAME: ${{ github.actor }}
+      VERBOSE: "true"
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: ${{ inputs.platform }}
+          image-type: "prod"
+          python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+          use-uv: ${{ inputs.use-uv }}
+          make-mnt-writeable-and-cleanup: true
+        id: breeze
+      - name: "Run airflowctl integration tests"
+        run: breeze testing airflow-ctl-integration-tests

Dockerfile

@@ -1116,6 +1116,7 @@ function install_from_sources() {
               --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \
               --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \
               --editable ./task-sdk-tests \
+              --editable ./airflow-ctl-tests \
               --editable ./devel-common[all] --editable ./dev \
               --group dev --group docs --group docs-gen --group leveldb"
         local -a projects_with_devel_dependencies

Dockerfile.ci

@@ -870,6 +870,7 @@ function install_from_sources() {
               --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \
               --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \
               --editable ./task-sdk-tests \
+              --editable ./airflow-ctl-tests \
               --editable ./devel-common[all] --editable ./dev \
               --group dev --group docs --group docs-gen --group leveldb"
         local -a projects_with_devel_dependencies

airflow-core/docs/howto/docker-compose/docker-compose.yaml

@@ -51,6 +51,8 @@ x-airflow-common:
   # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
   image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|}
   # build: .
+  env_file:
+    - ${ENV_FILE_PATH:-.env}
   environment:
     &airflow-common-env
     AIRFLOW__CORE__EXECUTOR: CeleryExecutor

airflow-ctl-tests/pyproject.toml

@@ -0,0 +1,64 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[build-system]
+requires = [ "hatchling==1.27.0" ]
+build-backend = "hatchling.build"
+
+[project]
+name = "apache-airflow-ctl-tests"
+description = "Airflow CTL tests for Apache Airflow"
+classifiers = [
+    "Private :: Do Not Upload",
+]
+requires-python = ">=3.10,!=3.13"
+authors = [
+    { name = "Apache Software Foundation", email = "dev@airflow.apache.org" },
+]
+maintainers = [
+    { name = "Apache Software Foundation", email="dev@airflow.apache.org" },
+]
+version = "0.0.1"
+
+dependencies = [
+    "apache-airflow-ctl",
+    "apache-airflow-devel-common",
+]
+
+[tool.pytest.ini_options]
+addopts = "-rasl --verbosity=2 -p no:flaky -p no:nose -p no:legacypath"
+norecursedirs = [
+    ".eggs",
+]
+log_level = "INFO"
+filterwarnings = [
+    "error::pytest.PytestCollectionWarning",
+]
+python_files = [
+    "*.py",
+]
+
+# Keep temporary directories (created by `tmp_path`) for 2 recent runs only failed tests.
+tmp_path_retention_count = "2"
+tmp_path_retention_policy = "failed"
+
+[tool.hatch.build.targets.sdist]
+exclude = ["*"]
+
+[tool.hatch.build.targets.wheel]
+bypass-selection = true

airflow-ctl-tests/tests/airflowctl_tests/__init__.py

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from rich.console import Console
+
+console = Console(width=400, color_system="standard")
+
+
+__all__ = ["console"]