From 9fee08ddf9d26dc458e0a78ddb8030240f65b645 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 29 Jun 2025 16:25:19 +0100 Subject: [PATCH 1/5] Filter only provided integration paths --- .../airflow_breeze/commands/testing_commands.py | 1 + dev/breeze/src/airflow_breeze/utils/run_tests.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 7f88b8bffaf44..3efcb54353b21 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -216,6 +216,7 @@ def _run_test( parallel_test_types_list=shell_params.parallel_test_types_list, keep_env_variables=shell_params.keep_env_variables, no_db_cleanup=shell_params.no_db_cleanup, + integration=shell_params.integration, ) pytest_args.extend(extra_pytest_args) # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 74cc0c6281975..32af4e8d7d4d0 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -245,14 +245,26 @@ def convert_test_type_to_pytest_args( *, test_group: GroupOfTests, test_type: str, + integration: tuple | None = None, ) -> list[str]: if test_type == "None": return [] if test_type in ALL_TEST_SUITES: - return [ + all_paths = [ *TEST_GROUP_TO_TEST_FOLDERS[test_group], *ALL_TEST_SUITES[test_type], ] + + if integration: + filtered_paths = [ + path + for path in all_paths + if any(path.endswith(f"{value}/tests/integration") for value in integration) + ] + + return filtered_paths + return all_paths + if test_group == GroupOfTests.SYSTEM and test_type != NONE_TEST_TYPE: get_console().print(f"[error]Only {NONE_TEST_TYPE} should be allowed as test type[/]") sys.exit(1) @@ -336,6 +348,7 @@ def generate_args_for_pytest( python_version: str, keep_env_variables: bool, no_db_cleanup: bool, + integration: tuple | None = None, ): result_log_file, warnings_file, coverage_file = test_paths(test_type, backend) if skip_db_tests and parallel_test_types_list: @@ -347,6 +360,7 @@ def generate_args_for_pytest( args = convert_test_type_to_pytest_args( test_group=test_group, test_type=test_type, + integration=integration, ) args.extend( [ From 1ce837b0615022a249422600bb2b603a2b1c2541 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 29 Jun 2025 16:53:48 +0100 Subject: [PATCH 2/5] Fix tests --- dev/breeze/src/airflow_breeze/utils/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 32af4e8d7d4d0..62f9463257c43 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -255,7 +255,7 @@ def convert_test_type_to_pytest_args( *ALL_TEST_SUITES[test_type], ] - if integration: + if integration and test_group == GroupOfTests.INTEGRATION_PROVIDERS: filtered_paths = [ path for path in all_paths From 6f6dd32902f6ffd26558c643ca13a22ea11074ac Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 29 Jun 2025 17:00:04 +0100 Subject: [PATCH 3/5] Rename gremline integration name to tinkerpop --- contributing-docs/testing/integration_tests.rst | 4 ++-- dev/breeze/doc/images/output-commands.svg | 4 ++-- dev/breeze/doc/images/output_shell.svg | 4 ++-- dev/breeze/doc/images/output_shell.txt | 2 +- dev/breeze/doc/images/output_start-airflow.svg | 4 ++-- dev/breeze/doc/images/output_start-airflow.txt | 2 +- .../doc/images/output_testing_providers-integration-tests.svg | 4 ++-- .../doc/images/output_testing_providers-integration-tests.txt | 2 +- dev/breeze/src/airflow_breeze/global_constants.py | 2 +- .../{integration-gremlin.yml => integration-tinkerpop.yml} | 0 10 files changed, 14 insertions(+), 14 deletions(-) rename scripts/ci/docker-compose/{integration-gremlin.yml => integration-tinkerpop.yml} (100%) diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index bc7e9c70f1d32..401b383ab3bc9 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -64,8 +64,6 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | drill | Integration required for drill operator and hook. | +--------------+-------------------------------------------------------+ -| gremlin | Integration required for gremlin operator and hook. | -+--------------+-------------------------------------------------------+ | kafka | Integration required for Kafka hooks. | +--------------+-------------------------------------------------------+ | kerberos | Integration that provides Kerberos authentication. | @@ -88,6 +86,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | statsd | Integration required for Statsd hooks. | +--------------+-------------------------------------------------------+ +| tinkerpop | Integration required for gremlin operator and hook. | ++--------------+-------------------------------------------------------+ | trino | Integration required for Trino hooks. | +--------------+-------------------------------------------------------+ | ydb | Integration required for YDB tests. | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 1ba42bc55607f..df5d3aa538068 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -332,9 +332,9 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ +│kafka | kerberos | keycloak | mongo | mssql |          │ │openlineage | otel | pinot | qdrant | redis | statsd | │ -│trino | ydb)                                           │ +│tinkerpop | trino | ydb)                               │ │--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ │r(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 6010232f0caee..bc3a6baecce93 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -586,9 +586,9 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ +│kafka | kerberos | keycloak | mongo | mssql |          │ │openlineage | otel | pinot | qdrant | redis | statsd | │ -│trino | ydb)                                           │ +│tinkerpop | trino | ydb)                               │ │--load-example-dags-eEnable configuration to load example DAGs when starting│ │Airflow.                                               │ │--load-default-connections-cEnable configuration to load default connections when  │ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index 3c0d8fc50b9bf..50018caa69fa6 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -7ce73872a6ef26151af64d2a850104f6 +74abdb31412e7ee706db5f3ece5beabc diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 071838e1e0ca0..d7f9cf7b2c694 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -497,9 +497,9 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ +│kafka | kerberos | keycloak | mongo | mssql |          │ │openlineage | otel | pinot | qdrant | redis | statsd | │ -│trino | ydb)                                           │ +│tinkerpop | trino | ydb)                               │ │--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ │r(required for Airflow 3).                              │ │[default: standalone-dag-processor]                    │ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 34b052e06659a..6b0006dd201e9 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -0ef2581aec890d0fa4cdd00b74c53049 +e5263cc1f97fc2bc3523f9d60d23dcc6 diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index b46e3ae7edd29..335eeb609c09b 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ │--integrationProviders Integration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | drill | gremlin | kafka | mongo | mssql | openlineage | │ -│pinot | qdrant | redis | trino | ydb)                                                              │ +│(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql | openlineage | pinot |   │ +│qdrant | redis | tinkerpop | trino | ydb)                                                          │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index 601b2a047983e..7209f31d3a698 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -9f2c3376aabee8cc8f009519d3c4f878 +cafc7c9fe7699b3b14f0d15b9d331fb1 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index afa0f4dae3b6f..401387dbddb70 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -68,7 +68,7 @@ "celery", "cassandra", "drill", - "gremlin", + "tinkerpop", "kafka", "mongo", "mssql", diff --git a/scripts/ci/docker-compose/integration-gremlin.yml b/scripts/ci/docker-compose/integration-tinkerpop.yml similarity index 100% rename from scripts/ci/docker-compose/integration-gremlin.yml rename to scripts/ci/docker-compose/integration-tinkerpop.yml From eb89bb056e8a4c0f2250e788d6ab756b88de3d49 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 29 Jun 2025 17:05:16 +0100 Subject: [PATCH 4/5] Fix selective_checks test --- dev/breeze/tests/test_selective_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4b16252ad0619..f014eb662d21a 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1145,7 +1145,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "testable-core-integrations": "['kerberos']", - "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'gremlin', 'kafka', " + "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'tinkerpop', 'kafka', " "'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", "needs-mypy": "true", "mypy-checks": ALL_MYPY_CHECKS, From 5677d045f0427ca277a9a6c846371c30830f75f2 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 29 Jun 2025 17:13:14 +0100 Subject: [PATCH 5/5] Update @integration pytest marker with tinkerpop --- .../tests/integration/apache/tinkerpop/hooks/test_gremlin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py index 2a2ea25f95625..507de5663cea5 100644 --- a/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py +++ b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py @@ -26,7 +26,7 @@ AIRFLOW_CONN_GREMLIN_DEFAULT = "ws://mylogin:mysecret@gremlin:8182/gremlin" -@pytest.mark.integration("gremlin") +@pytest.mark.integration("tinkerpop") class TestGremlinHook: def setup_method(self): os.environ["AIRFLOW_CONN_GREMLIN_DEFAULT"] = AIRFLOW_CONN_GREMLIN_DEFAULT