diff --git a/Dockerfile.ci b/Dockerfile.ci index 0927b49eb7ba3..4e39ccf966f95 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1208,6 +1208,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566" + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index a5d6c6deb873e..af0554c48ab23 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -70,6 +70,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | keycloak | Integration for manual testing of multi-team Airflow. | +--------------+-------------------------------------------------------+ +| localstack | Integration that emulates AWS services locally. | ++--------------+-------------------------------------------------------+ | mongo | Integration required for MongoDB hooks. | +--------------+-------------------------------------------------------+ | mssql | Integration required for mssql hooks. | diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index 38d9db39d830a..4d0d4c6905be1 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -392,6 +392,29 @@ These are all available flags of ``start-airflow`` command: :width: 100% :alt: Breeze start-airflow +Running External System Integrations with Breeze +------------------------------------------------ + +You can run Airflow alongside external systems in Breeze, such as Kafka, Cassandra, MongoDB, and more. + +To start Airflow with an integration, use the following command: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration + +For example, to run Airflow with Kafka: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration kafka + +Check the available integrations by running: + +.. code-block:: bash + + breeze --integration --help + Launching multiple terminals in the same environment ---------------------------------------------------- diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index d3656bc530f4e..4df859dfffe88 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -338,11 +338,11 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ -│openlineage | otel | pinot | qdrant | redis | redis |  │ -│statsd | tinkerpop | trino | ydb)                      │ -│--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ -│r(required for Airflow 3).                              │ +│kafka | kerberos | keycloak | localstack | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis |  │ +│redis | statsd | tinkerpop | trino | ydb)              │ +│--standalone-dag-processor/--no-standalone-dag-process…Run standalone dag processor for start-airflow         │ +│(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ │--auth-managerSpecify the auth manager to set       │ │(>SimpleAuthManager< | FabAuthManager)│ diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 6dbf70093a2bf..89e7b8ddcc04d 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -604,15 +604,15 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ -│openlineage | otel | pinot | qdrant | redis | redis |  │ -│statsd | tinkerpop | trino | ydb)                      │ +│kafka | kerberos | keycloak | localstack | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis |  │ +│redis | statsd | tinkerpop | trino | ydb)              │ │--load-example-dags-eEnable configuration to load example DAGs when starting│ │Airflow.                                               │ │--load-default-connections-cEnable configuration to load default connections when  │ │starting Airflow.                                      │ -│--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ -│r(required for Airflow 3).                              │ +│--standalone-dag-processor/--no-standalone-dag-process…Run standalone dag processor for start-airflow         │ +│(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ │--start-api-server-with-examplesStart minimal airflow api-server with examples (for    │ │testing purposes) when entering breeze.                │ @@ -706,8 +706,8 @@ │--excluded-providersJSON-string of dictionary containing excluded providers │ │per python version ({'3.12': ['provider']})             │ │(TEXT)                                                  │ -│--install-airflow-with-constraints/--no-install-airflow-Install airflow in a separate step, with constraints    │ -│with-constraintsdetermined from package or airflow version.             │ +│--install-airflow-with-constraints/--no-install-airflow…Install airflow in a separate step, with constraints    │ +│determined from package or airflow version.             │ │[default: install-airflow-with-constraints]             │ │--install-selected-providersComma-separated list of providers selected to be        │ │installed (implies --use-distributions-from-dist).      │ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index a2bca2d863a95..6ca99f63129ff 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -c23fcef0bfb6f2bbb46937d673e4cddb +11c2a41c8c4dd97becbe2af203fb9baa diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index e4f8058584b2b..7bd5899d0228c 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -518,11 +518,11 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ -│openlineage | otel | pinot | qdrant | redis | redis |  │ -│statsd | tinkerpop | trino | ydb)                      │ -│--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ -│r(required for Airflow 3).                              │ +│kafka | kerberos | keycloak | localstack | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis |  │ +│redis | statsd | tinkerpop | trino | ydb)              │ +│--standalone-dag-processor/--no-standalone-dag-process…Run standalone dag processor for start-airflow         │ +│(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ │--auth-managerSpecify the auth manager to set       │ │(>SimpleAuthManager< | FabAuthManager)│ @@ -598,8 +598,8 @@ │[default: ""]                                           │ │--clean-airflow-installationClean the airflow installation before installing version│ │specified by --use-airflow-version.                     │ -│--install-airflow-with-constraints/--no-install-airflow-Install airflow in a separate step, with constraints    │ -│with-constraintsdetermined from package or airflow version.             │ +│--install-airflow-with-constraints/--no-install-airflow…Install airflow in a separate step, with constraints    │ +│determined from package or airflow version.             │ │[default: install-airflow-with-constraints]             │ │--install-selected-providersComma-separated list of providers selected to be        │ │installed (implies --use-distributions-from-dist).      │ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 70645b7317e13..44878609bd098 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -b2b6b3d14961062524f862a86299ccce +9da141373d346a15dd576fa386acbb3b diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index 6bee01df22dbe..d023b849a3f65 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ │--integrationProviders Integration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql | openlineage | pinot |   │ -│qdrant | redis | tinkerpop | trino | ydb)                                                          │ +│(all | all-testable | cassandra | celery | drill | kafka | localstack | mongo | mssql | openlineage│ +│| pinot | qdrant | redis | tinkerpop | trino | ydb)                                                │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index 7a3452bc580f7..f8e31d8449611 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -66a5d51390e696c06b63d6ad296c7784 +c01e190daa309e88d3021a63f3099b52 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index b4dc7002df59a..3327b13c042af 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -78,6 +78,7 @@ "drill", "tinkerpop", "kafka", + "localstack", "mongo", "mssql", "pinot", @@ -88,6 +89,7 @@ ] DISABLE_TESTABLE_INTEGRATIONS_FROM_CI = [ "mssql", + "localstack", # just for local integration testing for now ] DISABLE_TESTABLE_INTEGRATIONS_FROM_ARM = [ "kerberos", diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6c43d29d76697..082b3838f75d0 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -429,6 +429,7 @@ def compose_file(self) -> str: else: integrations = self.integration for integration in integrations: + get_console().print(f"[info]Adding integration compose file for {integration}[/]") compose_file_list.append(DOCKER_COMPOSE_DIR / f"integration-{integration}.yml") if "trino" in integrations and "kerberos" not in integrations: get_console().print( diff --git a/scripts/ci/docker-compose/integration-localstack.yml b/scripts/ci/docker-compose/integration-localstack.yml new file mode 100644 index 0000000000000..68560bb047698 --- /dev/null +++ b/scripts/ci/docker-compose/integration-localstack.yml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" + image: localstack/localstack:4.7 + labels: + breeze.description: "Integration that emulates AWS services locally." + ports: + - "4566:4566" # LocalStack Gateway + - "4510-4559:4510-4559" # external services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + + airflow: + environment: + - INTEGRATION_LOCALSTACK=true + depends_on: + - localstack diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index af93ccaa026f7..bf64cb795e973 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -184,6 +184,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566" + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags