diff --git a/Dockerfile.ci b/Dockerfile.ci index 230b25ba8aee9..61eab57f2d1e3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1208,6 +1208,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566" + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index a5d6c6deb873e..af0554c48ab23 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -70,6 +70,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | keycloak | Integration for manual testing of multi-team Airflow. | +--------------+-------------------------------------------------------+ +| localstack | Integration that emulates AWS services locally. | ++--------------+-------------------------------------------------------+ | mongo | Integration required for MongoDB hooks. | +--------------+-------------------------------------------------------+ | mssql | Integration required for mssql hooks. | diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index 38d9db39d830a..4d0d4c6905be1 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -392,6 +392,29 @@ These are all available flags of ``start-airflow`` command: :width: 100% :alt: Breeze start-airflow +Running External System Integrations with Breeze +------------------------------------------------ + +You can run Airflow alongside external systems in Breeze, such as Kafka, Cassandra, MongoDB, and more. + +To start Airflow with an integration, use the following command: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration + +For example, to run Airflow with Kafka: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration kafka + +Check the available integrations by running: + +.. code-block:: bash + + breeze --integration --help + Launching multiple terminals in the same environment ---------------------------------------------------- diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index eaa5000bec732..857b530a7d954 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -331,62 +331,62 @@ Usage:breeze[OPTIONS] COMMAND [ARGS]... ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for   │ +│--python-pPython major/minor version used in Airflow image for   │ │images.                                                │ │(>3.10< | 3.11 | 3.12 | 3.13)                          │ │[default: 3.10]                                        │ -│--integrationCore Integrations to enable when running (can be more  │ +│--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ -│openlineage | otel | pinot | qdrant | redis | redis |  │ -│statsd | tinkerpop | trino | ydb)                      │ -│--standalone-dag-processor/--no-standalone-dag-process…Run standalone dag processor for start-airflow         │ -│(required for Airflow 3).                              │ +│kafka | kerberos | keycloak | localstack | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis |  │ +│redis | statsd | tinkerpop | trino | ydb)              │ +│--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ +│r(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ -│--auth-managerSpecify the auth manager to set       │ +│--auth-managerSpecify the auth manager to set       │ │(>SimpleAuthManager< | FabAuthManager)│ │[default: SimpleAuthManager]          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Docker Compose selection and cleanup ───────────────────────────────────────────────────────────────────────────────╮ -│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy breeze       │ -│project name and you can use `breeze down --project-name docker-compose` to stop all containers   │ +│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy breeze       │ +│project name and you can use `breeze down --project-name docker-compose` to stop all containers   │ │belonging to it.                                                                                  │ │(breeze | prek | docker-compose)                                                                  │ │[default: breeze]                                                                                 │ -│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` option is   │ +│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` option is   │ │ignored when building images.                                                                     │ │(TEXT)                                                                                            │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Database ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--backend-bDatabase backend to use. Default is 'sqlite'. If 'none' is chosen, Breeze will│ +│--backend-bDatabase backend to use. Default is 'sqlite'. If 'none' is chosen, Breeze will│ │start with an invalid database configuration — no database will be available, │ │and any attempt to run Airflow will fail. Use 'none' only for specific non-DB │ │test cases.                                                                   │ │(>sqlite< | mysql | postgres | none)                                          │ │[default: sqlite]                                                             │ -│--postgres-version-PVersion of Postgres used.(>13< | 14 | 15 | 16 | 17)[default: 13]│ -│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ -│--db-reset-d/--no-db-resetReset DB when entering the container.[default: no-db-reset]│ +│--postgres-version-PVersion of Postgres used.(>13< | 14 | 15 | 16 | 17)[default: 13]│ +│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ +│--db-reset-d/--no-db-resetReset DB when entering the container.[default: no-db-reset]│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Build CI image (before entering shell) ─────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ │[default: autodetect]                                         │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ │[default: 300; x>=1]                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Other options ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--forward-credentials-fForward local credentials to container when running.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│--forward-credentials-fForward local credentials to container when running.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ │(INTEGER RANGE)                                                                       │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ │start-airflow          Enter breeze environment and starts all Airflow components in the tmux session. Compile    │ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index a52f06728d2f5..691681a156caf 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -5bdfe1d4afe64fb8bd3e07d5c6e9c2d5 +345a25e9b7fd41a8e1d14d2c275e387d diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index fe487fdf0937a..49b8a2470417e 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -da22448455ba68de7ee5d2b1bd206265 +c5ecfff87a01e315b8f558798a5dc35c diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index 6bee01df22dbe..d023b849a3f65 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ │--integrationProviders Integration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql | openlineage | pinot |   │ -│qdrant | redis | tinkerpop | trino | ydb)                                                          │ +│(all | all-testable | cassandra | celery | drill | kafka | localstack | mongo | mssql | openlineage│ +│| pinot | qdrant | redis | tinkerpop | trino | ydb)                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index 7a3452bc580f7..f8e31d8449611 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -66a5d51390e696c06b63d6ad296c7784 +c01e190daa309e88d3021a63f3099b52 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 8d688366aa5ce..3af544999c954 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -72,6 +72,7 @@ "drill", "tinkerpop", "kafka", + "localstack", "mongo", "mssql", "pinot", @@ -82,6 +83,7 @@ ] DISABLE_TESTABLE_INTEGRATIONS_FROM_CI = [ "mssql", + "localstack", # just for local integration testing for now ] KEYCLOAK_INTEGRATION = "keycloak" STATSD_INTEGRATION = "statsd" diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6c43d29d76697..082b3838f75d0 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -429,6 +429,7 @@ def compose_file(self) -> str: else: integrations = self.integration for integration in integrations: + get_console().print(f"[info]Adding integration compose file for {integration}[/]") compose_file_list.append(DOCKER_COMPOSE_DIR / f"integration-{integration}.yml") if "trino" in integrations and "kerberos" not in integrations: get_console().print( diff --git a/scripts/ci/docker-compose/integration-localstack.yml b/scripts/ci/docker-compose/integration-localstack.yml new file mode 100644 index 0000000000000..68560bb047698 --- /dev/null +++ b/scripts/ci/docker-compose/integration-localstack.yml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" + image: localstack/localstack:4.7 + labels: + breeze.description: "Integration that emulates AWS services locally." + ports: + - "4566:4566" # LocalStack Gateway + - "4510-4559:4510-4559" # external services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + + airflow: + environment: + - INTEGRATION_LOCALSTACK=true + depends_on: + - localstack diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index af93ccaa026f7..bf64cb795e973 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -184,6 +184,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566" + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags